feat(remote): add archive snapshot provenance

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## v0.13.5 - Unreleased
 
 - Require notarized, single-architecture `crawlctl` release binaries bound to the repository-pinned signed tag and protected remote `main` commit, restore signing-keychain state before credential-free candidate probes, roll back partial artifact promotion, and keep naturally quarantined Gatekeeper execution plus Full Disk Access continuity as separate clean-VM gates because raw binaries and ZIP submission carriers cannot carry stapled tickets and raw-CLI `spctl` assessment is not an app-launch verdict.
+- Add immutable SQLite snapshot bundle manifests with deterministic source and representation digests, digest-scoped compressed objects and chunks, and explicit snapshot publishing helpers while preserving the legacy `current.*` bundle layout.
 
 ## v0.13.4 - 2026-07-09
 

remote/contract.go

@@ -22,6 +22,7 @@ const (
 type Contract struct {
 	ProtocolVersion string      `json:"protocol_version"`
 	Service         string      `json:"service,omitempty"`
+	ReleaseSHA      string      `json:"release_sha,omitempty"`
 	Routes          []RouteSpec `json:"routes"`
 	Apps            []AppSpec   `json:"apps"`
 	Auth            []AuthSpec  `json:"auth,omitempty"`
@@ -74,6 +75,7 @@ func BaseContract() Contract {
 			{Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/query", Auth: AuthReader},
 			{Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/batch-read", Auth: AuthReader},
 			{Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/ingest", Auth: AuthPublisher},
+			{Method: http.MethodPost, Path: "/v1/apps/:app/archives/:archive/cutover", Auth: AuthPublisher},
 			{Method: http.MethodPut, Path: "/v1/apps/:app/archives/:archive/sqlite", Auth: AuthPublisher},
 		},
 		Auth: []AuthSpec{

remote/remote.go

@@ -185,46 +185,98 @@ type Identity struct {
 	Roles []string `json:"roles,omitempty"`
 }
 
+type ArchiveSnapshot struct {
+	ID                 string `json:"id"`
+	SourceSHA256       string `json:"source_sha256,omitempty"`
+	SchemaName         string `json:"schema_name,omitempty"`
+	SchemaVersion      int    `json:"schema_version,omitempty"`
+	SchemaHash         string `json:"schema_hash,omitempty"`
+	SourceSyncAt       string `json:"source_sync_at,omitempty"`
+	DatasetGeneratedAt string `json:"dataset_generated_at,omitempty"`
+	CoverageComplete   bool   `json:"coverage_complete,omitempty"`
+	PublishedAt        string `json:"published_at,omitempty"`
+	CutoverAt          string `json:"cutover_at,omitempty"`
+}
+
+type ArchivePublish struct {
+	Status     string `json:"status"`
+	SnapshotID string `json:"snapshot_id,omitempty"`
+	StartedAt  string `json:"started_at,omitempty"`
+}
+
 type Archive struct {
-	ID            string   `json:"id"`
-	App           string   `json:"app"`
-	Slug          string   `json:"slug"`
-	SchemaName    string   `json:"schema_name,omitempty"`
-	SchemaVersion int      `json:"schema_version,omitempty"`
-	SchemaHash    string   `json:"schema_hash,omitempty"`
-	Capabilities  []string `json:"capabilities,omitempty"`
-	LastIngestAt  string   `json:"last_ingest_at,omitempty"`
-	LastSyncAt    string   `json:"last_sync_at,omitempty"`
+	ID            string           `json:"id"`
+	App           string           `json:"app"`
+	Slug          string           `json:"slug"`
+	SchemaName    string           `json:"schema_name,omitempty"`
+	SchemaVersion int              `json:"schema_version,omitempty"`
+	SchemaHash    string           `json:"schema_hash,omitempty"`
+	Capabilities  []string         `json:"capabilities,omitempty"`
+	LastIngestAt  string           `json:"last_ingest_at,omitempty"`
+	LastSyncAt    string           `json:"last_sync_at,omitempty"`
+	Snapshot      *ArchiveSnapshot `json:"snapshot,omitempty"`
+	Publish       *ArchivePublish  `json:"publish,omitempty"`
 }
 
 type Status struct {
-	App          string          `json:"app"`
-	Archive      string          `json:"archive"`
-	Mode         string          `json:"mode,omitempty"`
-	GeneratedAt  string          `json:"generated_at,omitempty"`
-	LastSyncAt   string          `json:"last_sync_at,omitempty"`
-	LastIngestAt string          `json:"last_ingest_at,omitempty"`
-	Counts       []control.Count `json:"counts,omitempty"`
-	Capabilities []string        `json:"capabilities,omitempty"`
-	SQLiteObject *SQLiteObject   `json:"sqlite_object,omitempty"`
-	SQLiteBundle *SQLiteBundle   `json:"sqlite_bundle,omitempty"`
-	Warnings     []string        `json:"warnings,omitempty"`
+	App                string            `json:"app"`
+	Archive            string            `json:"archive"`
+	Mode               string            `json:"mode,omitempty"`
+	GeneratedAt        string            `json:"generated_at,omitempty"`
+	SchemaName         string            `json:"schema_name,omitempty"`
+	SchemaVersion      int               `json:"schema_version,omitempty"`
+	SchemaHash         string            `json:"schema_hash,omitempty"`
+	LastSyncAt         string            `json:"last_sync_at,omitempty"`
+	LastIngestAt       string            `json:"last_ingest_at,omitempty"`
+	Counts             []control.Count   `json:"counts,omitempty"`
+	Capabilities       []string          `json:"capabilities,omitempty"`
+	SQLiteObject       *SQLiteObject     `json:"sqlite_object,omitempty"`
+	SQLiteBundle       *SQLiteBundle     `json:"sqlite_bundle,omitempty"`
+	SnapshotMode       string            `json:"snapshot_mode,omitempty"`
+	SnapshotCutoverAt  string            `json:"snapshot_cutover_at,omitempty"`
+	ActiveSnapshotID   string            `json:"active_snapshot_id,omitempty"`
+	SourceSyncAt       string            `json:"source_sync_at,omitempty"`
+	DatasetGeneratedAt string            `json:"dataset_generated_at,omitempty"`
+	CoverageComplete   bool              `json:"coverage_complete,omitempty"`
+	Datasets           []DatasetCoverage `json:"datasets,omitempty"`
+	Snapshot           *ArchiveSnapshot  `json:"snapshot,omitempty"`
+	Publish            *ArchivePublish   `json:"publish,omitempty"`
+	Warnings           []string          `json:"warnings,omitempty"`
+}
+
+type DatasetCoverage struct {
+	Dataset            string `json:"dataset"`
+	RowCount           int64  `json:"row_count,omitempty"`
+	EligibleCount      int64  `json:"eligible_count,omitempty"`
+	CoveredCount       int64  `json:"covered_count,omitempty"`
+	FreshCount         int64  `json:"fresh_count,omitempty"`
+	MaxSourceAt        string `json:"max_source_at,omitempty"`
+	DatasetGeneratedAt string `json:"dataset_generated_at,omitempty"`
+	Complete           bool   `json:"complete,omitempty"`
 }
 
 type QueryRequest struct {
-	App     string         `json:"app,omitempty"`
-	Archive string         `json:"archive,omitempty"`
-	Name    string         `json:"name"`
-	Args    map[string]any `json:"args,omitempty"`
-	Limit   int            `json:"limit,omitempty"`
-	Cursor  string         `json:"cursor,omitempty"`
+	App        string         `json:"app,omitempty"`
+	Archive    string         `json:"archive,omitempty"`
+	Name       string         `json:"name"`
+	Args       map[string]any `json:"args,omitempty"`
+	Limit      int            `json:"limit,omitempty"`
+	Cursor     string         `json:"cursor,omitempty"`
+	SnapshotID string         `json:"snapshot_id,omitempty"`
 }
 
 type QueryStats struct {
-	RowsRead    int64  `json:"rows_read,omitempty"`
-	RowsWritten int64  `json:"rows_written,omitempty"`
-	DurationMS  int64  `json:"duration_ms,omitempty"`
-	ServedBy    string `json:"served_by,omitempty"`
+	RowsRead           int64  `json:"rows_read,omitempty"`
+	RowsWritten        int64  `json:"rows_written,omitempty"`
+	DurationMS         int64  `json:"duration_ms,omitempty"`
+	ServedBy           string `json:"served_by,omitempty"`
+	SnapshotID         string `json:"snapshot_id,omitempty"`
+	SourceSyncAt       string `json:"source_sync_at,omitempty"`
+	DatasetGeneratedAt string `json:"dataset_generated_at,omitempty"`
+	CoverageComplete   bool   `json:"coverage_complete,omitempty"`
+	SchemaVersion      int    `json:"schema_version,omitempty"`
+	ObservationOrder   string `json:"observation_order,omitempty"`
+	NextCursor         string `json:"next_cursor,omitempty"`
 }
 
 type QueryResult struct {
@@ -234,38 +286,52 @@ type QueryResult struct {
 	Cursor     string           `json:"cursor,omitempty"`
 	Stats      QueryStats       `json:"stats,omitempty"`
 	SchemaHash string           `json:"schema_hash,omitempty"`
+	Snapshot   *ArchiveSnapshot `json:"snapshot,omitempty"`
 }
 
 type IngestManifest struct {
-	App           string `json:"app"`
-	Archive       string `json:"archive"`
-	SchemaName    string `json:"schema_name,omitempty"`
-	SchemaVersion int    `json:"schema_version"`
-	SchemaHash    string `json:"schema_hash"`
-	Mode          string `json:"mode,omitempty"`
-	Source        string `json:"source,omitempty"`
-	SourceSyncAt  string `json:"source_sync_at,omitempty"`
+	App           string   `json:"app"`
+	Archive       string   `json:"archive"`
+	SchemaName    string   `json:"schema_name,omitempty"`
+	SchemaVersion int      `json:"schema_version"`
+	SchemaHash    string   `json:"schema_hash"`
+	Mode          string   `json:"mode,omitempty"`
+	Source        string   `json:"source,omitempty"`
+	SourceSyncAt  string   `json:"source_sync_at,omitempty"`
+	SnapshotID    string   `json:"snapshot_id,omitempty"`
+	SourceSHA256  string   `json:"source_sha256,omitempty"`
+	Capabilities  []string `json:"capabilities,omitempty"`
 }
 
 type IngestRequest struct {
-	Manifest IngestManifest `json:"manifest"`
-	Table    string         `json:"table"`
-	Columns  []string       `json:"columns"`
-	Rows     [][]any        `json:"rows"`
-	Cursor   string         `json:"cursor,omitempty"`
-	Final    bool           `json:"final,omitempty"`
+	Manifest      IngestManifest `json:"manifest"`
+	Table         string         `json:"table"`
+	Columns       []string       `json:"columns"`
+	Rows          [][]any        `json:"rows"`
+	Cursor        string         `json:"cursor,omitempty"`
+	MutationToken string         `json:"mutation_token,omitempty"`
+	Final         bool           `json:"final,omitempty"`
 }
 
 type IngestResult struct {
 	RunID           string `json:"run_id,omitempty"`
 	Table           string `json:"table,omitempty"`
+	SnapshotID      string `json:"snapshot_id,omitempty"`
+	MutationToken   string `json:"mutation_token,omitempty"`
 	RowsAccepted    int64  `json:"rows_accepted,omitempty"`
 	Cursor          string `json:"cursor,omitempty"`
 	Complete        bool   `json:"complete,omitempty"`
 	ResetIncomplete bool   `json:"reset_incomplete,omitempty"`
 	ResetDeleted    int64  `json:"reset_deleted,omitempty"`
 }
 
+type CutoverResult struct {
+	Archive      string `json:"archive,omitempty"`
+	SnapshotID   string `json:"snapshot_id"`
+	SnapshotMode string `json:"snapshot_mode,omitempty"`
+	CutoverAt    string `json:"cutover_at,omitempty"`
+}
+
 type SQLiteUploadRequest struct {
 	Body          io.Reader
 	Size          int64
@@ -400,6 +466,14 @@ func (c *Client) Ingest(ctx context.Context, app, archive string, req IngestRequ
 	return out, err
 }
 
+func (c *Client) Cutover(ctx context.Context, app, archive, snapshotID string) (CutoverResult, error) {
+	var out CutoverResult
+	err := c.do(ctx, http.MethodPost, archivePath(app, archive, "cutover"), struct {
+		SnapshotID string `json:"snapshot_id"`
+	}{SnapshotID: strings.TrimSpace(snapshotID)}, &out, true)
+	return out, err
+}
+
 func (c *Client) UploadSQLite(ctx context.Context, app, archive string, upload SQLiteUploadRequest) (SQLiteUploadResult, error) {
 	if upload.Body == nil {
 		return SQLiteUploadResult{}, errors.New("sqlite upload body is required")
@@ -420,18 +494,25 @@ func (c *Client) UploadSQLiteBundlePart(ctx context.Context, app, archive string
 	if part.Body == nil {
 		return SQLiteUploadResult{}, errors.New("sqlite bundle part body is required")
 	}
+	if part.SnapshotID != "" && !validSQLiteSnapshotID(part.SnapshotID) {
+		return SQLiteUploadResult{}, errors.New("sqlite bundle snapshot id must be empty or a lowercase sha256 digest")
+	}
 	headers := http.Header{}
 	headers.Set("content-type", "application/gzip")
 	headers.Set("x-crawl-sqlite-upload", "bundle-part")
 	headers.Set("x-crawl-bundle-part-index", fmt.Sprintf("%d", part.Index))
 	setHeader(headers, "x-crawl-content-sha256", part.SHA256)
 	setHeader(headers, "x-crawl-compression", part.Compression)
+	setHeader(headers, "x-crawl-snapshot-id", part.SnapshotID)
 	var out SQLiteUploadResult
 	err := c.doRaw(ctx, http.MethodPut, archivePath(app, archive, "sqlite"), part.Body, part.Size, headers, &out, true)
 	return out, err
 }
 
 func (c *Client) UploadSQLiteBundleFiles(ctx context.Context, app, archive string, manifest SQLiteBundleManifest, parts []SQLiteBundlePartFile) (SQLiteBundleUploadResult, error) {
+	if manifest.SnapshotID != "" && !validSQLiteSnapshotID(manifest.SnapshotID) {
+		return SQLiteBundleUploadResult{}, errors.New("sqlite bundle snapshot id must be empty or a lowercase sha256 digest")
+	}
 	for _, part := range parts {
 		file, err := os.Open(part.Path)
 		if err != nil {
@@ -443,6 +524,7 @@ func (c *Client) UploadSQLiteBundleFiles(ctx context.Context, app, archive strin
 			Size:        part.Size,
 			SHA256:      part.SHA256,
 			Compression: SQLiteGzipCompression,
+			SnapshotID:  manifest.SnapshotID,
 		})
 		_ = file.Close()
 		if uploadErr != nil {
@@ -453,6 +535,9 @@ func (c *Client) UploadSQLiteBundleFiles(ctx context.Context, app, archive strin
 }
 
 func (c *Client) UploadSQLiteBundleManifest(ctx context.Context, app, archive string, manifest SQLiteBundleManifest) (SQLiteBundleUploadResult, error) {
+	if manifest.SnapshotID != "" && !validSQLiteSnapshotID(manifest.SnapshotID) {
+		return SQLiteBundleUploadResult{}, errors.New("sqlite bundle snapshot id must be empty or a lowercase sha256 digest")
+	}
 	if strings.TrimSpace(manifest.App) == "" {
 		manifest.App = strings.TrimSpace(app)
 	}