From 791cf0d0fbea9007ad88b78d3c596e4f95184fce Mon Sep 17 00:00:00 2001 From: adu-crypto <94821467+adu-crypto@users.noreply.github.com> Date: Thu, 1 Sep 2022 16:42:23 +0800 Subject: [PATCH] feat: Add a cli cmd to prune old states according to current settings (#12742) * add PruningCmd and change PruneStores signature * the mimimum default pruning interval is 10 Co-authored-by: Marko (cherry picked from commit d874acee4cf1a32a9078967ed35807a8eb647609) # Conflicts: # CHANGELOG.md # store/rootmulti/store.go --- CHANGELOG.md | 11 ++++ client/pruning/main.go | 117 +++++++++++++++++++++++++++++++++++++++ simapp/simd/cmd/root.go | 2 + store/rootmulti/store.go | 71 ++++++++++++++++++++++-- 4 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 client/pruning/main.go diff --git a/CHANGELOG.md b/CHANGELOG.md index feaca9011e7b..4618354444b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,18 @@ Ref: https://keepachangelog.com/en/1.0.0/ ## [Unreleased] +<<<<<<< HEAD ## v0.45.8 - 2022-08-25 +======= +### Features + +* (x/authz) [#12648](https://github.com/cosmos/cosmos-sdk/pull/12648) Add an allow list, an optional list of addresses allowed to receive bank assests via authz MsgSend grant. +* (sdk.Coins) [#12627](https://github.com/cosmos/cosmos-sdk/pull/12627) Make a Denoms method on sdk.Coins. +* (testutil) [#12973](https://github.com/cosmos/cosmos-sdk/pull/12973) Add generic `testutil.RandSliceElem` function which selects a random element from the list. +* (client) [#12936](https://github.com/cosmos/cosmos-sdk/pull/12936) Add capability to preprocess transactions before broadcasting from a higher level chain. +* (x/authz) [#13047](https://github.com/cosmos/cosmos-sdk/pull/13047) Add a GetAuthorization function to the keeper. +* (cli) [#12742](https://github.com/cosmos/cosmos-sdk/pull/12742) Add the `prune` CLI cmd to manually prune app store history versions based on the pruning options. +>>>>>>> d874acee4 (feat: Add a cli cmd to prune old states according to current settings (#12742)) ### Improvements diff --git a/client/pruning/main.go b/client/pruning/main.go new file mode 100644 index 000000000000..fb62dd21a050 --- /dev/null +++ b/client/pruning/main.go @@ -0,0 +1,117 @@ +package pruning + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/cosmos/cosmos-sdk/client/flags" + pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types" + "github.com/cosmos/cosmos-sdk/server" + servertypes "github.com/cosmos/cosmos-sdk/server/types" + "github.com/cosmos/cosmos-sdk/store/rootmulti" + "github.com/tendermint/tendermint/libs/log" + dbm "github.com/tendermint/tm-db" +) + +const FlagAppDBBackend = "app-db-backend" + +// PruningCmd prunes the sdk root multi store history versions based on the pruning options +// specified by command flags. +func PruningCmd(appCreator servertypes.AppCreator) *cobra.Command { + cmd := &cobra.Command{ + Use: "prune", + Short: "Prune app history states by keeping the recent heights and deleting old heights", + Long: `Prune app history states by keeping the recent heights and deleting old heights. + The pruning option is provided via the '--pruning' flag or alternatively with '--pruning-keep-recent' + + For '--pruning' the options are as follows: + + default: the last 362880 states are kept + nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node) + everything: 2 latest states will be kept + custom: allow pruning options to be manually specified through 'pruning-keep-recent'. + besides pruning options, database home directory and database backend type should also be specified via flags + '--home' and '--app-db-backend'. + valid app-db-backend type includes 'goleveldb', 'cleveldb', 'rocksdb', 'boltdb', and 'badgerdb'. + `, + Example: "prune --home './' --app-db-backend 'goleveldb' --pruning 'custom' --pruning-keep-recent 100", + RunE: func(cmd *cobra.Command, _ []string) error { + vp := viper.New() + + // Bind flags to the Context's Viper so we can get pruning options. + if err := vp.BindPFlags(cmd.Flags()); err != nil { + return err + } + pruningOptions, err := server.GetPruningOptionsFromFlags(vp) + if err != nil { + return err + } + fmt.Printf("get pruning options from command flags, strategy: %v, keep-recent: %v\n", + pruningOptions.Strategy, + pruningOptions.KeepRecent, + ) + + home := vp.GetString(flags.FlagHome) + db, err := openDB(home, server.GetAppDBBackend(vp)) + if err != nil { + return err + } + + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + app := appCreator(logger, db, nil, vp) + cms := app.CommitMultiStore() + + rootMultiStore, ok := cms.(*rootmulti.Store) + if !ok { + return fmt.Errorf("currently only support the pruning of rootmulti.Store type") + } + latestHeight := rootmulti.GetLatestVersion(db) + // valid heights should be greater than 0. + if latestHeight <= 0 { + return fmt.Errorf("the database has no valid heights to prune, the latest height: %v", latestHeight) + } + + var pruningHeights []int64 + for height := int64(1); height < latestHeight; height++ { + if height < latestHeight-int64(pruningOptions.KeepRecent) { + pruningHeights = append(pruningHeights, height) + } + } + if len(pruningHeights) == 0 { + fmt.Printf("no heights to prune\n") + return nil + } + fmt.Printf( + "pruning heights start from %v, end at %v\n", + pruningHeights[0], + pruningHeights[len(pruningHeights)-1], + ) + + err = rootMultiStore.PruneStores(false, pruningHeights) + if err != nil { + return err + } + fmt.Printf("successfully pruned the application root multi stores\n") + return nil + }, + } + + cmd.Flags().String(flags.FlagHome, "", "The database home directory") + cmd.Flags().String(FlagAppDBBackend, "", "The type of database for application and snapshots databases") + cmd.Flags().String(server.FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") + cmd.Flags().Uint64(server.FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint64(server.FlagPruningInterval, 10, + `Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom'), + this is not used by this command but kept for compatibility with the complete pruning options`) + + return cmd +} + +func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) { + dataDir := filepath.Join(rootDir, "data") + return dbm.NewDB("application", backendType, dataDir) +} diff --git a/simapp/simd/cmd/root.go b/simapp/simd/cmd/root.go index 3189241f9b8d..f4afb2c89138 100644 --- a/simapp/simd/cmd/root.go +++ b/simapp/simd/cmd/root.go @@ -19,6 +19,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/debug" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/client/keys" + "github.com/cosmos/cosmos-sdk/client/pruning" "github.com/cosmos/cosmos-sdk/client/rpc" "github.com/cosmos/cosmos-sdk/server" servertypes "github.com/cosmos/cosmos-sdk/server/types" @@ -152,6 +153,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig params.EncodingConfig) { testnetCmd(simapp.ModuleBasics, banktypes.GenesisBalancesIterator{}), debug.Cmd(), config.Cmd(), + pruning.PruningCmd(newApp), ) a := appCreator{encodingConfig} diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 1f8ef097d0f2..987f9e84bd31 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -156,7 +156,7 @@ func (rs *Store) GetStores() map[types.StoreKey]types.CommitKVStore { // LoadLatestVersionAndUpgrade implements CommitMultiStore func (rs *Store) LoadLatestVersionAndUpgrade(upgrades *types.StoreUpgrades) error { - ver := getLatestVersion(rs.db) + ver := GetLatestVersion(rs.db) return rs.loadVersion(ver, upgrades) } @@ -167,7 +167,7 @@ func (rs *Store) LoadVersionAndUpgrade(ver int64, upgrades *types.StoreUpgrades) // LoadLatestVersion implements CommitMultiStore. func (rs *Store) LoadLatestVersion() error { - ver := getLatestVersion(rs.db) + ver := GetLatestVersion(rs.db) return rs.loadVersion(ver, nil) } @@ -378,7 +378,7 @@ func (rs *Store) ListeningEnabled(key types.StoreKey) bool { func (rs *Store) LastCommitID() types.CommitID { if rs.lastCommitInfo == nil { return types.CommitID{ - Version: getLatestVersion(rs.db), + Version: GetLatestVersion(rs.db), } } @@ -548,7 +548,66 @@ func (rs *Store) GetKVStore(key types.StoreKey) types.KVStore { return store } +<<<<<<< HEAD // GetStoreByName performs a lookup of a StoreKey given a store name typically +======= +func (rs *Store) handlePruning(version int64) error { + rs.pruningManager.HandleHeight(version - 1) // we should never prune the current version. + if !rs.pruningManager.ShouldPruneAtHeight(version) { + return nil + } + rs.logger.Info("prune start", "height", version) + defer rs.logger.Info("prune end", "height", version) + return rs.PruneStores(true, nil) +} + +// PruneStores prunes the specific heights of the multi store. +// If clearPruningManager is true, the pruning manager will return the pruning heights, +// and they are appended to the pruningHeights to be pruned. +func (rs *Store) PruneStores(clearPruningManager bool, pruningHeights []int64) (err error) { + if clearPruningManager { + heights, err := rs.pruningManager.GetFlushAndResetPruningHeights() + if err != nil { + return err + } + + if len(heights) == 0 { + rs.logger.Debug("no heights to be pruned from pruning manager") + } + + pruningHeights = append(pruningHeights, heights...) + } + + if len(pruningHeights) == 0 { + rs.logger.Debug("no heights need to be pruned") + return nil + } + + rs.logger.Debug("pruning heights", "heights", pruningHeights) + + for key, store := range rs.stores { + // If the store is wrapped with an inter-block cache, we must first unwrap + // it to get the underlying IAVL store. + if store.GetStoreType() != types.StoreTypeIAVL { + continue + } + + store = rs.GetCommitKVStore(key) + + err := store.(*iavl.Store).DeleteVersions(pruningHeights...) + if err == nil { + continue + } + + if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist { + return err + } + } + return nil +} + +// getStoreByName performs a lookup of a StoreKey given a store name typically +>>>>>>> d874acee4 (feat: Add a cli cmd to prune old states according to current settings (#12742)) // provided in a path. The StoreKey is then used to perform a lookup and return // a Store. If the Store is wrapped in an inter-block cache, it will be unwrapped // prior to being returned. If the StoreKey does not exist, nil is returned. @@ -661,7 +720,11 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error { if height == 0 { return sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot snapshot height 0") } +<<<<<<< HEAD if height > uint64(rs.LastCommitID().Version) { +======= + if height > uint64(GetLatestVersion(rs.db)) { +>>>>>>> d874acee4 (feat: Add a cli cmd to prune old states according to current settings (#12742)) return sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot snapshot future height %v", height) } @@ -926,7 +989,7 @@ type storeParams struct { initialVersion uint64 } -func getLatestVersion(db dbm.DB) int64 { +func GetLatestVersion(db dbm.DB) int64 { bz, err := db.Get([]byte(latestVersionKey)) if err != nil { panic(err)