From a4e8050251fdcacd1139b8a6d78e443328dd3399 Mon Sep 17 00:00:00 2001 From: Anil Kumar Date: Sun, 26 Apr 2026 20:27:54 -0700 Subject: [PATCH] =?UTF-8?q?The=20Raft=20consensus=20loop=20took=20the=20by?= =?UTF-8?q?tes=20of=20a=20committed=20log=20entry=20(or=20snapshot)=20and?= =?UTF-8?q?=20called=20protoutil.UnmarshalBlockOrPanic(...)=20on=20them.?= =?UTF-8?q?=20That=20function=20does=20what=20its=20name=20says=20?= =?UTF-8?q?=E2=80=94=20if=20the=20bytes=20don't=20decode=20as=20a=20common?= =?UTF-8?q?.Block=20protobuf,=20it=20raises=20a=20Go=20panic,=20which=20te?= =?UTF-8?q?rminates=20=20=20=20the=20entire=20orderer=20process.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Anil Kumar --- orderer/consensus/etcdraft/chain.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/orderer/consensus/etcdraft/chain.go b/orderer/consensus/etcdraft/chain.go index 88dad1a62b3..6e91a359181 100644 --- a/orderer/consensus/etcdraft/chain.go +++ b/orderer/consensus/etcdraft/chain.go @@ -252,7 +252,10 @@ func NewChain( var snapBlkNum uint64 var cc raftpb.ConfState if s := storage.Snapshot(); !raft.IsEmptySnap(s) { - b := protoutil.UnmarshalBlockOrPanic(s.Data) + b, err := protoutil.UnmarshalBlock(s.Data) + if err != nil { + return nil, errors.Wrap(err, "failed to unmarshal block from raft snapshot") + } snapBlkNum = b.Header.Number cc = s.Metadata.ConfState } @@ -1189,7 +1192,17 @@ func (c *Chain) apply(ents []raftpb.Entry) { break } - block := protoutil.UnmarshalBlockOrPanic(ents[i].Data) + block, err := protoutil.UnmarshalBlock(ents[i].Data) + if err != nil { + // A committed Raft entry that does not unmarshal as a Block indicates either + // storage corruption or a byzantine proposer. Skipping the entry would advance + // appliedIndex past data that other followers may apply successfully, silently + // forking this node's state. Halt this chain so an operator can investigate; + // other channels' chains in this orderer process continue running. + c.logger.Errorf("Failed to unmarshal committed block at raft index %d: %s; halting chain", ents[i].Index, err) + go c.halt() + return + } c.writeBlock(block, ents[i].Index) c.Metrics.CommittedBlockNumber.Set(float64(block.Header.Number)) @@ -1250,7 +1263,15 @@ func (c *Chain) apply(ents []raftpb.Entry) { // at position==0, ents[position].Type is ambiguous, it can be either of {raftpb.EntryNormal, raftpb.EntryConfChange} // take a snapshot only for ents[position].Type == raftpb.EntryNormal if c.accDataSize >= c.sizeLimit && ents[position].Type == raftpb.EntryNormal && len(ents[position].Data) > 0 { - b := protoutil.UnmarshalBlockOrPanic(ents[position].Data) + b, err := protoutil.UnmarshalBlock(ents[position].Data) + if err != nil { + // We just successfully unmarshalled and applied this same entry above; if we + // fail here, memory has been corrupted between then and now. Halt the chain + // rather than panic the orderer process. + c.logger.Errorf("Failed to unmarshal block at raft index %d while preparing snapshot: %s; halting chain", ents[position].Index, err) + go c.halt() + return + } select { case c.gcC <- &gc{index: c.appliedIndex, state: c.confState, data: ents[position].Data}: c.logger.Infof("Accumulated %d bytes since last snapshot, exceeding size limit (%d bytes), "+