CLI tools for the DAGStore inverted index

api/api_storage.go

@@ -212,7 +212,13 @@ type StorageMiner interface {
 
 	// IndexerAnnounceDeal informs indexer nodes that a new deal was received,
 	// so they can download its index
-	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error
+	IndexerAnnounceDeal(ctx context.Context, proposalCid cid.Cid) error //perm:admin
+
+	// DagstoreInvertedIndexSize returns the size of the inverted index.
+	DagstoreInvertedIndexSize(ctx context.Context) (int64, error) //perm:admin
+
+	// DagstoreLookupPieces returns information about shards that contain the given CID.
+	DagstoreLookupPieces(ctx context.Context, cid string) ([]DagstoreShardInfo, error) //perm:admin
 
 	// RuntimeSubsystems returns the subsystems that are enabled
 	// in this instance.

cmd/lotus-miner/dagstore.go

@@ -21,6 +21,8 @@ var dagstoreCmd = &cli.Command{
 		dagstoreRecoverShardCmd,
 		dagstoreInitializeAllCmd,
 		dagstoreGcCmd,
+		dagstoreInvertedIndexSizeCmd,
+		dagstoreLookupPiecesCmd,
 	},
 }
 
@@ -52,38 +54,7 @@ var dagstoreListShardsCmd = &cli.Command{
 			return err
 		}
 
-		if len(shards) == 0 {
-			return nil
-		}
-
-		tw := tablewriter.New(
-			tablewriter.Col("Key"),
-			tablewriter.Col("State"),
-			tablewriter.Col("Error"),
-		)
-
-		colors := map[string]color.Attribute{
-			"ShardStateAvailable": color.FgGreen,
-			"ShardStateServing":   color.FgBlue,
-			"ShardStateErrored":   color.FgRed,
-			"ShardStateNew":       color.FgYellow,
-		}
-
-		for _, s := range shards {
-			m := map[string]interface{}{
-				"Key": s.Key,
-				"State": func() string {
-					if c, ok := colors[s.State]; ok {
-						return color.New(c).Sprint(s.State)
-					}
-					return s.State
-				}(),
-				"Error": s.Error,
-			}
-			tw.Write(m)
-		}
-
-		return tw.Flush(os.Stdout)
+		return printTableShards(shards)
 	},
 }
 
@@ -265,3 +236,108 @@ var dagstoreGcCmd = &cli.Command{
 		return nil
 	},
 }
+
+func printTableShards(shards []api.DagstoreShardInfo) error {
+	if len(shards) == 0 {
+		return nil
+	}
+
+	tw := tablewriter.New(
+		tablewriter.Col("Key"),
+		tablewriter.Col("State"),
+		tablewriter.Col("Error"),
+	)
+
+	colors := map[string]color.Attribute{
+		"ShardStateAvailable": color.FgGreen,
+		"ShardStateServing":   color.FgBlue,
+		"ShardStateErrored":   color.FgRed,
+		"ShardStateNew":       color.FgYellow,
+	}
+
+	for _, s := range shards {
+		m := map[string]interface{}{
+			"Key": s.Key,
+			"State": func() string {
+				if c, ok := colors[s.State]; ok {
+					return color.New(c).Sprint(s.State)
+				}
+				return s.State
+			}(),
+			"Error": s.Error,
+		}
+		tw.Write(m)
+	}
+	return tw.Flush(os.Stdout)
+}
+
+var dagstoreInvertedIndexSizeCmd = &cli.Command{
+	Name:  "inverted-index-size",
+	Usage: "Inspect the dagstore inverted index size",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		size, err := marketsApi.DagstoreInvertedIndexSize(ctx)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(size)
+
+		return nil
+	},
+}
+
+var dagstoreLookupPiecesCmd = &cli.Command{
+	Name:  "lookup-pieces",
+	Usage: "Lookup pieces that a given CID belongs to",
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "color",
+			Usage:       "use color in display output",
+			DefaultText: "depends on output being a TTY",
+		},
+		&cli.BoolFlag{
+			Name:        "cid",
+			Usage:       "cid to lookup",
+			DefaultText: "",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.IsSet("color") {
+			color.NoColor = !cctx.Bool("color")
+		}
+
+		marketsApi, closer, err := lcli.GetMarketsAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		shards, err := marketsApi.DagstoreLookupPieces(ctx, cctx.String("cid"))
+		if err != nil {
+			return err
+		}
+
+		return printTableShards(shards)
+	},
+}

documentation/en/api-v0-methods-miner.md

@@ -22,7 +22,9 @@
   * [DagstoreGC](#DagstoreGC)
   * [DagstoreInitializeAll](#DagstoreInitializeAll)
   * [DagstoreInitializeShard](#DagstoreInitializeShard)
+  * [DagstoreInvertedIndexSize](#DagstoreInvertedIndexSize)
   * [DagstoreListShards](#DagstoreListShards)
+  * [DagstoreLookupPieces](#DagstoreLookupPieces)
   * [DagstoreRecoverShard](#DagstoreRecoverShard)
 * [Deals](#Deals)
   * [DealsConsiderOfflineRetrievalDeals](#DealsConsiderOfflineRetrievalDeals)
@@ -433,6 +435,16 @@ Inputs:
 
 Response: `{}`
 
+### DagstoreInvertedIndexSize
+DagstoreInvertedIndexSize returns the size of the inverted index.
+
+
+Perms: admin
+
+Inputs: `null`
+
+Response: `9`
+
 ### DagstoreListShards
 DagstoreListShards returns information about all shards known to the
 DAG store. Only available on nodes running the markets subsystem.
@@ -444,6 +456,21 @@ Inputs: `null`
 
 Response: `null`
 
+### DagstoreLookupPieces
+DagstoreLookupPieces returns information about shards that contain the given CID.
+
+
+Perms: admin
+
+Inputs:
+```json
+[
+  "string value"
+]
+```
+
+Response: `null`
+
 ### DagstoreRecoverShard
 DagstoreRecoverShard attempts to recover a failed shard.
 
@@ -673,7 +700,7 @@ IndexerAnnounceDeal informs indexer nodes that a new deal was received,
 so they can download its index
 
 
-Perms: 
+Perms: admin
 
 Inputs:
 ```json

documentation/en/cli-lotus-miner.md

@@ -1031,12 +1031,14 @@ USAGE:
    lotus-miner dagstore command [command options] [arguments...]
 
 COMMANDS:
-   list-shards       List all shards known to the dagstore, with their current status
-   initialize-shard  Initialize the specified shard
-   recover-shard     Attempt to recover a shard in errored state
-   initialize-all    Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default
-   gc                Garbage collect the dagstore
-   help, h           Shows a list of commands or help for one command
+   list-shards          List all shards known to the dagstore, with their current status
+   initialize-shard     Initialize the specified shard
+   recover-shard        Attempt to recover a shard in errored state
+   initialize-all       Initialize all uninitialized shards, streaming results as they're produced; only shards for unsealed pieces are initialized by default
+   gc                   Garbage collect the dagstore
+   inverted-index-size  Inspect the dagstore inverted index size
+   lookup-pieces        Lookup pieces that a given CID belongs to
+   help, h              Shows a list of commands or help for one command
 
 OPTIONS:
    --help, -h  show help (default: false)
@@ -1115,6 +1117,35 @@ OPTIONS:
 
 ```
 
+### lotus-miner dagstore inverted-index-size
+```
+NAME:
+   lotus-miner dagstore inverted-index-size - Inspect the dagstore inverted index size
+
+USAGE:
+   lotus-miner dagstore inverted-index-size [command options] [arguments...]
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --help, -h  show help (default: false)
+
+```
+
+### lotus-miner dagstore lookup-pieces
+```
+NAME:
+   lotus-miner dagstore lookup-pieces - Lookup pieces that a given CID belongs to
+
+USAGE:
+   lotus-miner dagstore lookup-pieces [command options] [arguments...]
+
+OPTIONS:
+   --color     use color in display output (default: depends on output being a TTY)
+   --cid       cid to lookup (default: false)
+   --help, -h  show help (default: false)
+
+```
+
 ## lotus-miner index
 ```
 NAME: