feat: trim heavy fields from raw_header before storage

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/cockroachdb/pebble v1.1.5
 	github.com/filecoin-project/go-jsonrpc v0.10.1
 	github.com/google/orderedcode v0.0.1
+	github.com/gorilla/websocket v1.4.2
 	github.com/prometheus/client_golang v1.23.2
 	github.com/rs/zerolog v1.34.0
 	github.com/spf13/cobra v1.10.2
@@ -50,7 +51,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/ipfs/go-log/v2 v2.0.8 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect

pkg/backfill/db/source.go

@@ -2,6 +2,8 @@ package db
 
 import (
 	"context"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -189,7 +191,7 @@ func (s *Source) FetchHeight(_ context.Context, height uint64, namespaces []type
 		Hash:      meta.BlockHash,
 		DataHash:  block.DataHash,
 		Time:      block.Time,
-		RawHeader: rawBlock,
+		RawHeader: buildMinimalRawHeader(height, block.Time, block.DataHash, meta.BlockHash),
 	}
 
 	if len(namespaces) == 0 {
@@ -710,6 +712,30 @@ func splitIntoParts(raw []byte, partSize int) [][]byte {
 	return out
 }
 
+// buildMinimalRawHeader synthesizes a small JSON object from the decoded block
+// fields. The backfill source reads raw protobuf, so we build the JSON that
+// consumers (ev-node) expect rather than storing the full protobuf blob.
+func buildMinimalRawHeader(height uint64, t time.Time, dataHash, blockHash []byte) []byte {
+	obj := map[string]any{
+		"header": map[string]any{
+			"height":    fmt.Sprintf("%d", height),
+			"time":      t.Format(time.RFC3339Nano),
+			"data_hash": hex.EncodeToString(dataHash),
+		},
+		"commit": map[string]any{
+			"height": fmt.Sprintf("%d", height),
+			"block_id": map[string]any{
+				"hash": hex.EncodeToString(blockHash),
+			},
+		},
+	}
+	raw, err := json.Marshal(obj)
+	if err != nil {
+		return nil
+	}
+	return raw
+}
+
 func writeTestBlock(path, backend, layout string, height uint64, hash, dataHash []byte, t time.Time, txs [][]byte, partSize int) error {
 	w, err := openWritable(path, backend)
 	if err != nil {

pkg/fetch/celestia_app.go

@@ -213,7 +213,10 @@ func mapBlockResponse(blockID *cometpb.BlockID, block *cometpb.Block) (*types.He
 	hdr := block.Header
 	t := hdr.Time.AsTime()
 
-	raw, err := json.Marshal(block)
+	// Store only the header sub-object, not the full block (which includes
+	// last_commit signatures, evidence, and transaction data — all stored
+	// separately or unused).
+	raw, err := json.Marshal(hdr)
 	if err != nil {
 		return nil, fmt.Errorf("marshal raw header: %w", err)
 	}

pkg/fetch/celestia_node.go

@@ -400,10 +400,49 @@ func mapHeader(raw json.RawMessage) (*types.Header, error) {
 		Hash:      []byte(h.Commit.BlockID.Hash),
 		DataHash:  []byte(h.Header.DataHash),
 		Time:      t,
-		RawHeader: []byte(raw),
+		RawHeader: TrimRawHeader([]byte(raw)),
 	}, nil
 }
 
+// heavyHeaderKeys are top-level ExtendedHeader fields that are large and unused
+// by any consumer. Removing them reduces per-header storage from ~87KB to ~2KB.
+//
+//   - dah: Data Availability Header with row/column roots (~64KB, 73%)
+//   - validator_set: full validator public keys and voting power (~14KB, 16%)
+//   - commit.signatures: individual validator signatures (~10KB, 12%)
+var heavyHeaderKeys = []string{"dah", "validator_set"}
+
+// TrimRawHeader removes large, unused fields from a Celestia ExtendedHeader JSON
+// to reduce storage footprint. The remaining ~1KB "header" object plus "commit"
+// metadata are sufficient for all known consumers (ev-node only reads the timestamp).
+func TrimRawHeader(raw []byte) []byte {
+	var obj map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &obj); err != nil {
+		return raw
+	}
+
+	for _, key := range heavyHeaderKeys {
+		delete(obj, key)
+	}
+
+	// Keep commit.block_id and commit.height but strip commit.signatures.
+	if commitRaw, ok := obj["commit"]; ok {
+		var commit map[string]json.RawMessage
+		if err := json.Unmarshal(commitRaw, &commit); err == nil {
+			delete(commit, "signatures")
+			if trimmed, err := json.Marshal(commit); err == nil {
+				obj["commit"] = trimmed
+			}
+		}
+	}
+
+	trimmed, err := json.Marshal(obj)
+	if err != nil {
+		return raw
+	}
+	return trimmed
+}
+
 func mapBlobs(raw json.RawMessage, height uint64) ([]types.Blob, error) {
 	// Celestia returns null/empty for no blobs.
 	if len(raw) == 0 || string(raw) == "null" {

pkg/fetch/celestia_node_test.go

@@ -231,6 +231,59 @@ func TestJsonInt64(t *testing.T) {
 	}
 }
 
+func TestTrimRawHeader(t *testing.T) {
+	raw := json.RawMessage(`{
+		"header": {"height":"100","time":"2025-01-01T00:00:00Z","chain_id":"test"},
+		"dah": {"row_roots":["AAAA","BBBB"],"column_roots":["CCCC","DDDD"]},
+		"validator_set": {"validators":[{"address":"abc","pub_key":{"type":"ed25519","value":"xxx"}}]},
+		"commit": {"height":"100","block_id":{"hash":"1234"},"signatures":[{"validator_address":"abc","signature":"sig"}]}
+	}`)
+
+	trimmed := TrimRawHeader(raw)
+	var obj map[string]json.RawMessage
+	if err := json.Unmarshal(trimmed, &obj); err != nil {
+		t.Fatalf("unmarshal trimmed: %v", err)
+	}
+
+	if _, ok := obj["dah"]; ok {
+		t.Error("dah should be removed")
+	}
+	if _, ok := obj["validator_set"]; ok {
+		t.Error("validator_set should be removed")
+	}
+	if _, ok := obj["header"]; !ok {
+		t.Error("header should be preserved")
+	}
+	if _, ok := obj["commit"]; !ok {
+		t.Fatal("commit should be preserved")
+	}
+
+	// commit should have block_id but not signatures
+	var commit map[string]json.RawMessage
+	if err := json.Unmarshal(obj["commit"], &commit); err != nil {
+		t.Fatalf("unmarshal commit: %v", err)
+	}
+	if _, ok := commit["block_id"]; !ok {
+		t.Error("commit.block_id should be preserved")
+	}
+	if _, ok := commit["signatures"]; ok {
+		t.Error("commit.signatures should be removed")
+	}
+
+	// Verify trimmed is much smaller
+	if len(trimmed) >= len(raw) {
+		t.Errorf("trimmed (%d bytes) should be smaller than original (%d bytes)", len(trimmed), len(raw))
+	}
+}
+
+func TestTrimRawHeaderInvalidJSON(t *testing.T) {
+	raw := []byte(`not json`)
+	trimmed := TrimRawHeader(raw)
+	if string(trimmed) != string(raw) {
+		t.Error("invalid JSON should be returned as-is")
+	}
+}
+
 func TestHexBytes(t *testing.T) {
 	tests := []struct {
 		input string