Skip to content

Commit e9840ad

Browse files
yperbasismh0lt
yperbasis
and
mh0lt
authoredMay 28, 2024··
add flag checking for pruning waypoints (#10508)
Cherry pick PR #10468 into the release branch Co-authored-by: Mark Holt <135143369+mh0lt@users.noreply.github.com>
·
v2.61.32.60.7
1 parent 608a940 commit e9840ad

File tree

5 files changed

+191
-152
lines changed

5 files changed

+191
-152
lines changed
 

‎core/rawdb/accessors_chain.go

-112
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121
"context"
2222
"encoding/binary"
2323
"encoding/json"
24-
"errors"
2524
"fmt"
2625
"math"
2726
"math/big"
@@ -42,7 +41,6 @@ import (
4241

4342
"github.com/ledgerwatch/erigon/core/types"
4443
"github.com/ledgerwatch/erigon/ethdb/cbor"
45-
"github.com/ledgerwatch/erigon/polygon/heimdall"
4644
"github.com/ledgerwatch/erigon/rlp"
4745
)
4846

@@ -1096,116 +1094,6 @@ func PruneBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int) error {
10961094
return nil
10971095
}
10981096

1099-
// PruneBorBlocks - delete [1, to) old blocks after moving it to snapshots.
1100-
// keeps genesis in db: [1, to)
1101-
// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
1102-
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
1103-
func PruneBorBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
1104-
c, err := tx.Cursor(kv.BorEventNums)
1105-
if err != nil {
1106-
return err
1107-
}
1108-
defer c.Close()
1109-
var blockNumBytes [8]byte
1110-
binary.BigEndian.PutUint64(blockNumBytes[:], blockTo)
1111-
k, v, err := c.Seek(blockNumBytes[:])
1112-
if err != nil {
1113-
return err
1114-
}
1115-
var eventIdTo uint64 = math.MaxUint64
1116-
if k != nil {
1117-
eventIdTo = binary.BigEndian.Uint64(v)
1118-
}
1119-
c1, err := tx.RwCursor(kv.BorEvents)
1120-
if err != nil {
1121-
return err
1122-
}
1123-
defer c1.Close()
1124-
counter := blocksDeleteLimit
1125-
for k, _, err = c1.First(); err == nil && k != nil && counter > 0; k, _, err = c1.Next() {
1126-
eventId := binary.BigEndian.Uint64(k)
1127-
if eventId >= eventIdTo {
1128-
break
1129-
}
1130-
if err = c1.DeleteCurrent(); err != nil {
1131-
return err
1132-
}
1133-
counter--
1134-
}
1135-
if err != nil {
1136-
return err
1137-
}
1138-
firstSpanToKeep := SpanIdAt(blockTo)
1139-
c2, err := tx.RwCursor(kv.BorSpans)
1140-
if err != nil {
1141-
return err
1142-
}
1143-
defer c2.Close()
1144-
counter = blocksDeleteLimit
1145-
for k, _, err := c2.First(); err == nil && k != nil && counter > 0; k, _, err = c2.Next() {
1146-
spanId := binary.BigEndian.Uint64(k)
1147-
if spanId >= firstSpanToKeep {
1148-
break
1149-
}
1150-
if err = c2.DeleteCurrent(); err != nil {
1151-
return err
1152-
}
1153-
counter--
1154-
}
1155-
1156-
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
1157-
if err != nil {
1158-
return err
1159-
}
1160-
1161-
defer checkpointCursor.Close()
1162-
lastCheckpointToRemove, err := heimdall.CheckpointIdAt(tx, blockTo)
1163-
1164-
if err != nil {
1165-
return err
1166-
}
1167-
1168-
var checkpointIdBytes [8]byte
1169-
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToRemove))
1170-
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Prev() {
1171-
if err = checkpointCursor.DeleteCurrent(); err != nil {
1172-
return err
1173-
}
1174-
}
1175-
1176-
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)
1177-
1178-
if err != nil {
1179-
return err
1180-
}
1181-
1182-
defer milestoneCursor.Close()
1183-
1184-
var lastMilestoneToRemove heimdall.MilestoneId
1185-
1186-
for blockCount := 1; err != nil && blockCount < blocksDeleteLimit; blockCount++ {
1187-
lastMilestoneToRemove, err = heimdall.MilestoneIdAt(tx, blockTo-uint64(blockCount))
1188-
1189-
if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
1190-
return err
1191-
} else {
1192-
if blockCount == blocksDeleteLimit-1 {
1193-
return nil
1194-
}
1195-
}
1196-
}
1197-
1198-
var milestoneIdBytes [8]byte
1199-
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToRemove))
1200-
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Prev() {
1201-
if err = milestoneCursor.DeleteCurrent(); err != nil {
1202-
return err
1203-
}
1204-
}
1205-
1206-
return nil
1207-
}
1208-
12091097
func TruncateCanonicalChain(ctx context.Context, db kv.RwTx, from uint64) error {
12101098
return db.ForEach(kv.HeaderCanonical, hexutility.EncodeTs(from), func(k, _ []byte) error {
12111099
return db.Delete(kv.HeaderCanonical, k)

‎core/rawdb/blockio/block_writer.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/ledgerwatch/erigon-lib/kv"
1414
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
1515
"github.com/ledgerwatch/erigon/core/rawdb"
16+
"github.com/ledgerwatch/erigon/polygon/bor/bordb"
1617
"github.com/ledgerwatch/log/v3"
1718
)
1819

@@ -116,5 +117,5 @@ func (w *BlockWriter) PruneBlocks(ctx context.Context, tx kv.RwTx, blockTo uint6
116117
// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
117118
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
118119
func (w *BlockWriter) PruneBorBlocks(ctx context.Context, tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
119-
return rawdb.PruneBorBlocks(tx, blockTo, blocksDeleteLimit, SpanIdAt)
120+
return bordb.PruneBorBlocks(tx, blockTo, blocksDeleteLimit, SpanIdAt)
120121
}

‎eth/stagedsync/stage_bor_heimdall.go

+44-39
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/ledgerwatch/erigon/polygon/bor/borcfg"
3131
"github.com/ledgerwatch/erigon/polygon/bor/finality"
3232
"github.com/ledgerwatch/erigon/polygon/bor/finality/whitelist"
33+
borsnaptype "github.com/ledgerwatch/erigon/polygon/bor/snaptype"
3334
"github.com/ledgerwatch/erigon/polygon/bor/valset"
3435
"github.com/ledgerwatch/erigon/polygon/heimdall"
3536
"github.com/ledgerwatch/erigon/turbo/services"
@@ -890,63 +891,67 @@ func BorHeimdallUnwind(u *UnwindState, ctx context.Context, _ *StageState, tx kv
890891
}
891892

892893
// Removing checkpoints
893-
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "checkpoints") {
894-
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
894+
if borsnaptype.CheckpointsEnabled() {
895+
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "checkpoints") {
896+
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
895897

896-
if err != nil {
897-
return err
898-
}
899-
900-
defer checkpointCursor.Close()
901-
lastCheckpointToKeep, err := heimdall.CheckpointIdAt(tx, u.UnwindPoint)
902-
hasCheckpoints := true
903-
904-
if err != nil {
905-
if !errors.Is(err, heimdall.ErrCheckpointNotFound) {
898+
if err != nil {
906899
return err
907900
}
908901

909-
hasCheckpoints = false
910-
}
902+
defer checkpointCursor.Close()
903+
lastCheckpointToKeep, err := heimdall.CheckpointIdAt(tx, u.UnwindPoint)
904+
hasCheckpoints := true
911905

912-
if hasCheckpoints {
913-
var checkpointIdBytes [8]byte
914-
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToKeep+1))
915-
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Next() {
916-
if err = checkpointCursor.DeleteCurrent(); err != nil {
906+
if err != nil {
907+
if !errors.Is(err, heimdall.ErrCheckpointNotFound) {
917908
return err
918909
}
919-
}
920-
}
921-
}
922910

923-
// Removing milestones
924-
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "milestones") {
925-
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)
911+
hasCheckpoints = false
912+
}
926913

927-
if err != nil {
928-
return err
914+
if hasCheckpoints {
915+
var checkpointIdBytes [8]byte
916+
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToKeep+1))
917+
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Next() {
918+
if err = checkpointCursor.DeleteCurrent(); err != nil {
919+
return err
920+
}
921+
}
922+
}
929923
}
924+
}
930925

931-
defer milestoneCursor.Close()
932-
lastMilestoneToKeep, err := heimdall.MilestoneIdAt(tx, u.UnwindPoint)
933-
hasMilestones := true
926+
if borsnaptype.MilestonesEnabled() {
927+
// Removing milestones
928+
if len(cfg.unwindTypes) == 0 || slices.Contains(cfg.unwindTypes, "milestones") {
929+
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)
934930

935-
if err != nil {
936-
if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
931+
if err != nil {
937932
return err
938933
}
939934

940-
hasMilestones = false
941-
}
935+
defer milestoneCursor.Close()
936+
lastMilestoneToKeep, err := heimdall.MilestoneIdAt(tx, u.UnwindPoint)
937+
hasMilestones := true
942938

943-
if hasMilestones {
944-
var milestoneIdBytes [8]byte
945-
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToKeep+1))
946-
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Next() {
947-
if err = milestoneCursor.DeleteCurrent(); err != nil {
939+
if err != nil {
940+
if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
948941
return err
949942
}
943+
944+
hasMilestones = false
945+
}
946+
947+
if hasMilestones {
948+
var milestoneIdBytes [8]byte
949+
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToKeep+1))
950+
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Next() {
951+
if err = milestoneCursor.DeleteCurrent(); err != nil {
952+
return err
953+
}
954+
}
950955
}
951956
}
952957
}

‎polygon/bor/bordb/prune.go

+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package bordb
2+
3+
import (
4+
"encoding/binary"
5+
"errors"
6+
"math"
7+
8+
"github.com/ledgerwatch/erigon-lib/kv"
9+
"github.com/ledgerwatch/erigon/polygon/bor/snaptype"
10+
"github.com/ledgerwatch/erigon/polygon/heimdall"
11+
)
12+
13+
// PruneBorBlocks - delete [1, to) old blocks after moving it to snapshots.
14+
// keeps genesis in db: [1, to)
15+
// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
16+
// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
17+
func PruneBorBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
18+
c, err := tx.Cursor(kv.BorEventNums)
19+
if err != nil {
20+
return err
21+
}
22+
defer c.Close()
23+
var blockNumBytes [8]byte
24+
binary.BigEndian.PutUint64(blockNumBytes[:], blockTo)
25+
k, v, err := c.Seek(blockNumBytes[:])
26+
if err != nil {
27+
return err
28+
}
29+
var eventIdTo uint64 = math.MaxUint64
30+
if k != nil {
31+
eventIdTo = binary.BigEndian.Uint64(v)
32+
}
33+
c1, err := tx.RwCursor(kv.BorEvents)
34+
if err != nil {
35+
return err
36+
}
37+
defer c1.Close()
38+
counter := blocksDeleteLimit
39+
for k, _, err = c1.First(); err == nil && k != nil && counter > 0; k, _, err = c1.Next() {
40+
eventId := binary.BigEndian.Uint64(k)
41+
if eventId >= eventIdTo {
42+
break
43+
}
44+
if err = c1.DeleteCurrent(); err != nil {
45+
return err
46+
}
47+
counter--
48+
}
49+
if err != nil {
50+
return err
51+
}
52+
firstSpanToKeep := SpanIdAt(blockTo)
53+
c2, err := tx.RwCursor(kv.BorSpans)
54+
if err != nil {
55+
return err
56+
}
57+
defer c2.Close()
58+
counter = blocksDeleteLimit
59+
for k, _, err := c2.First(); err == nil && k != nil && counter > 0; k, _, err = c2.Next() {
60+
spanId := binary.BigEndian.Uint64(k)
61+
if spanId >= firstSpanToKeep {
62+
break
63+
}
64+
if err = c2.DeleteCurrent(); err != nil {
65+
return err
66+
}
67+
counter--
68+
}
69+
70+
if snaptype.CheckpointsEnabled() {
71+
checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
72+
if err != nil {
73+
return err
74+
}
75+
76+
defer checkpointCursor.Close()
77+
lastCheckpointToRemove, err := heimdall.CheckpointIdAt(tx, blockTo)
78+
79+
if err != nil {
80+
return err
81+
}
82+
83+
var checkpointIdBytes [8]byte
84+
binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToRemove))
85+
for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Prev() {
86+
if err = checkpointCursor.DeleteCurrent(); err != nil {
87+
return err
88+
}
89+
}
90+
}
91+
92+
if snaptype.MilestonesEnabled() {
93+
milestoneCursor, err := tx.RwCursor(kv.BorMilestones)
94+
95+
if err != nil {
96+
return err
97+
}
98+
99+
defer milestoneCursor.Close()
100+
101+
var lastMilestoneToRemove heimdall.MilestoneId
102+
103+
for blockCount := 1; err != nil && blockCount < blocksDeleteLimit; blockCount++ {
104+
lastMilestoneToRemove, err = heimdall.MilestoneIdAt(tx, blockTo-uint64(blockCount))
105+
106+
if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
107+
return err
108+
} else {
109+
if blockCount == blocksDeleteLimit-1 {
110+
return nil
111+
}
112+
}
113+
}
114+
115+
var milestoneIdBytes [8]byte
116+
binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToRemove))
117+
for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Prev() {
118+
if err = milestoneCursor.DeleteCurrent(); err != nil {
119+
return err
120+
}
121+
}
122+
}
123+
124+
return nil
125+
}

‎polygon/bor/snaptype/types.go

+20
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,26 @@ func BorSnapshotTypes() []snaptype.Type {
423423
return []snaptype.Type{BorEvents, BorSpans}
424424
}
425425

426+
func CheckpointsEnabled() bool {
427+
for _, snapType := range BorSnapshotTypes() {
428+
if snapType.Enum() == BorCheckpoints.Enum() {
429+
return true
430+
}
431+
}
432+
433+
return false
434+
}
435+
436+
func MilestonesEnabled() bool {
437+
for _, snapType := range BorSnapshotTypes() {
438+
if snapType.Enum() == BorMilestones.Enum() {
439+
return true
440+
}
441+
}
442+
443+
return false
444+
}
445+
426446
func extractValueRange(ctx context.Context, table string, valueFrom, valueTo uint64, db kv.RoDB, collect func([]byte) error, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) {
427447
logEvery := time.NewTicker(20 * time.Second)
428448
defer logEvery.Stop()

0 commit comments

Comments
 (0)
Please sign in to comment.