core/rawdb: simple legacy receipt converter #24028
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ package rawdb | |
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "math" | ||
| "os" | ||
| "path/filepath" | ||
|
|
@@ -617,3 +618,116 @@ func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes [] | |
|
|
||
| return hashes, err | ||
| } | ||
|
|
||
| // convertLegacyFn takes a raw freezer entry in an older format and | ||
| // returns it in the new format. | ||
| type convertLegacyFn = func([]byte) ([]byte, error) | ||
|
|
||
| // MigrateTable processes the entries in a given table in sequence | ||
| // converting them to a new format if they're of an old format. | ||
| func (f *freezer) MigrateTable(kind string, convert convertLegacyFn) error { | ||
| if f.readonly { | ||
| return errReadOnly | ||
| } | ||
| f.writeLock.Lock() | ||
| defer f.writeLock.Unlock() | ||
|
|
||
| table, ok := f.tables[kind] | ||
| if !ok { | ||
| return errUnknownTable | ||
| } | ||
| // forEach iterates every entry in the table serially and in order, calling `fn` | ||
| // with the item as argument. If `fn` returns an error the iteration stops | ||
| // and that error will be returned. | ||
| forEach := func(t *freezerTable, offset uint64, fn func(uint64, []byte) error) error { | ||
| var ( | ||
| items = atomic.LoadUint64(&t.items) | ||
| batchSize = uint64(1024) | ||
| maxBytes = uint64(1024 * 1024) | ||
| ) | ||
| for i := offset; i < items; { | ||
| if i+batchSize > items { | ||
| batchSize = items - i | ||
| } | ||
| data, err := t.RetrieveItems(i, batchSize, maxBytes) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| for j, item := range data { | ||
| if err := fn(i+uint64(j), item); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| i += uint64(len(data)) | ||
| } | ||
| return nil | ||
| } | ||
| // TODO(s1na): This is a sanity-check since as of now no process does tail-deletion. But the migration | ||
| // process assumes no deletion at tail and needs to be modified to account for that. | ||
| if table.itemOffset > 0 || table.itemHidden > 0 { | ||
|
There was a problem hiding this comment. Any specific reason for rejecting this type of table? It's ok to keep this checking but would be nice to get rid of it eventually. There was a problem hiding this comment. I added this quick check to make sure the table hasn't been tail-trimmed. Because the migration process will not fully work for such a table right now and need some tweaks. I'll add a todo comment |
||
| return fmt.Errorf("migration not supported for tail-deleted freezers") | ||
| } | ||
| ancientsPath := filepath.Dir(table.index.Name()) | ||
| // Set up new dir for the migrated table, the content of which | ||
| // we'll at the end move over to the ancients dir. | ||
| migrationPath := filepath.Join(ancientsPath, "migration") | ||
| newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind], false) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| var ( | ||
| batch = newTable.newBatch() | ||
| out []byte | ||
| start = time.Now() | ||
| logged = time.Now() | ||
| offset = newTable.items | ||
| ) | ||
| if offset > 0 { | ||
| log.Info("found previous migration attempt", "migrated", offset) | ||
| } | ||
| // Iterate through entries and transform them | ||
| if err := forEach(table, offset, func(i uint64, blob []byte) error { | ||
| if i%10000 == 0 && time.Since(logged) > 16*time.Second { | ||
| log.Info("Processing legacy elements", "count", i, "elapsed", common.PrettyDuration(time.Since(start))) | ||
| logged = time.Now() | ||
| } | ||
| out, err = convert(blob) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if err := batch.AppendRaw(i, out); err != nil { | ||
| return err | ||
| } | ||
| return nil | ||
| }); err != nil { | ||
| return err | ||
| } | ||
| if err := batch.commit(); err != nil { | ||
s1na marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return err | ||
| } | ||
| log.Info("Replacing old table files with migrated ones", "elapsed", common.PrettyDuration(time.Since(start))) | ||
| // Release and delete old table files. Note this won't | ||
| // delete the index file. | ||
| table.releaseFilesAfter(0, true) | ||
|
|
||
| if err := newTable.Close(); err != nil { | ||
| return err | ||
| } | ||
| files, err := ioutil.ReadDir(migrationPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| // Move migrated files to ancients dir. | ||
| for _, f := range files { | ||
| // This will replace the old index file as a side-effect. | ||
| if err := os.Rename(filepath.Join(migrationPath, f.Name()), filepath.Join(ancientsPath, f.Name())); err != nil { | ||
|
There was a problem hiding this comment. One thing to be aware of, which I find pretty counter-intuitive, is that prints out So it looks to me like this shouldn't work, but I haven't tested it, maybe it does? There was a problem hiding this comment. That is counter-intuitive...however in this case it works because There was a problem hiding this comment. Yes, maybe add some comments about the file name? So that next time we won't be confused. There was a problem hiding this comment. The behavior of For example, But I never test it :P |
||
| return err | ||
| } | ||
| } | ||
| // Delete by now empty dir. | ||
| if err := os.Remove(migrationPath); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Whenever we check for "Is Mainnet?", we usually only care about the genesis hash, not checking the networkid. I think we can ignore the network id
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is there any downsides to checking it nevertheless?