forked from gwatts/dyndump
/
main.go
224 lines (173 loc) · 8.3 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
// Copyright 2016 Gareth Watts
// Licensed under an MIT license
// See the LICENSE file for details
/*
Command dyndump dumps or restores a single DynamoDB table to/from a JSON
formatted file or collection of S3 objects.
It supports parallel connections for increased throughput, and rate
limiting to a specified read or write capacity.
JSON is emitted as a stream of objects, one per item in the canonical format
used by the DynamoDB API. Each object has a key for each field name with a
value object holding the type and field value. Eg
{
"string-field": {"S": "string value"},
"number-field": {"N": "123"}
}
The following types are defined by the DynamoDB API:
* S - String
* N - Number (encoded in JSON as a string)
* B - Binary (a base64 encoded string)
* BOOL - Boolean
* NULL - Null
* SS - String set
* NS - Number set
* BS - Binary set
* L - List
* M - Map
AWS credentials will be read from EC2 metadata, ~/.aws/credentials or from
the following environment variables:
* AWS_REGION
* AWS_ACCESS_KEY_ID
* AWS_SECRET_ACCESS_KEY
Usage:
dyndump supports four commands:
DUMP
Usage: dyndump dump [--silent] [--no-progress] [-cmpr] [--filename | --stdout] [(--s3-bucket --s3-prefix)] TABLENAME
Dump a table to file or S3
Arguments:
TABLENAME="" Table name to dump from Dynamo
Options:
-c, --consistent-read=false Enable consistent reads (at 2x capacity use)
-f, --filename="" Filename to write data to.
--stdout=false If true then send the output to stdout
-m, --maxitems=0 Maximum number of items to dump. Set to 0 to process all items
-p, --parallel=5 Number of concurrent channels to open to DynamoDB
-r, --read-capacity=5 Average aggregate read capacity to use for scan (set to 0 for unlimited)
--s3-bucket="" S3 bucket name to upload to
--s3-prefix="" Path prefix to use to store data in S3 (eg. "backups/2016-04-01-12:25-")
--silent=false Set to true to disable all non-error output
--no-progress=false Set to true to disable the progress bar
LOAD
Usage: dyndump load [--silent] [--no-progress] [-mpw] (--filename | --stdin | (--s3-bucket --s3-prefix)) TABLENAME
Load a table dump from S3 or file to a DynamoDB table
Arguments:
TABLENAME="" Table name to load into
Options:
--allow-overwrite=false Set to true to overwrite any existing rows
-f, --filename="" Filename to read data from. Set to "-" for stdin
--stdin=false If true then read the dump data from stdin
-m, --maxitems=0 Maximum number of items to load. Set to 0 to process all items
-p, --parallel=4 Number of concurrent channels to open to DynamoDB
-w, --write-capacity=5 Average aggregate write capacity to use for load (set to 0 for unlimited)
--s3-bucket="" S3 bucket name to read from
--s3-prefix="" Path prefix to use to read data from S3 (eg. "backups/2016-04-01-12:25-")
--silent=false Set to true to disable all non-error output
--no-progress=false Set to true to disable the progress bar
INFO
Usage: dyndump info --s3-bucket --s3-prefix
Display backup metadata from an S3 backup
Options:
--s3-bucket="" S3 bucket name to read from
--s3-prefix="" Path prefix to use to read data from S3 (eg. "backups/2016-04-01-12:25-")
DELETE
Usage: dyndump delete [--silent] [--no-progress] --s3-bucket --s3-prefix [--force]
Delete a backup from S3
Options:
--s3-bucket="" S3 bucket name to delete from
--s3-prefix="" Path prefix to use to delete from S3 (eg. "backups/2016-04-01-12:25-")
--force=false Set to true to disable the delete prompt
--silent=false Set to true to disable all non-error output
--no-progress=false Set to true to disable the progress bar
*/
package main
import (
"fmt"
"os"
"time"
"github.com/jawher/mow.cli"
)
const (
maxParallel = 1000
statsFrequency = 2 * time.Second
)
func fail(format string, a ...interface{}) {
fmt.Fprintf(os.Stderr, format+"\n", a...)
cli.Exit(100)
}
func checkGTE(value, min int, flag string) {
if value < min {
fail("%s must be %d or greater", flag, min)
}
}
func checkLTE(value, max int, flag string) {
if value > max {
fail("%s must be %d or less", flag, max)
}
}
func main() {
app := cli.App("dyndump", "Dump and restore DynamoDB database tables")
app.LongDesc = "long desc goes here"
app.Command("dump", "Dump a table to file or S3", func(cmd *cli.Cmd) {
cmd.Spec = "[-cmpr] [--filename | --stdout] [(--s3-bucket --s3-prefix)] TABLENAME"
action := &dumper{
tableName: cmd.StringArg("TABLENAME", "", "Table name to dump from Dynamo"),
consistentRead: cmd.BoolOpt("c consistent-read", false, "Enable consistent reads (at 2x capacity use)"),
filename: cmd.StringOpt("f filename", "", "Filename to write data to."),
stdout: cmd.BoolOpt("stdout", false, "If true then send the output to stdout"),
maxItems: cmd.IntOpt("m maxitems", 0, "Maximum number of items to dump. Set to 0 to process all items"),
parallel: cmd.IntOpt("p parallel", 5, "Number of concurrent channels to open to DynamoDB"),
readCapacity: cmd.IntOpt("r read-capacity", 5, "Average aggregate read capacity to use for scan (set to 0 for unlimited)"),
s3BucketName: cmd.StringOpt("s3-bucket", "", "S3 bucket name to upload to"),
s3Prefix: cmd.StringOpt("s3-prefix", "", `Path prefix to use to store data in S3 (eg. "backups/2016-04-01-12:25-")`),
}
cmd.Before = func() {
checkGTE(*action.parallel, 1, "--parallel")
checkLTE(*action.parallel, maxParallel, "--parallel")
checkGTE(*action.maxItems, 0, "--max-items")
checkGTE(*action.readCapacity, 0, "--read-capacity")
if *action.filename == "" && !*action.stdout && *action.s3BucketName == "" {
fail("Either --filename/--stdout and/or --s3-bucket and --s3-prefix must be set")
}
}
cmd.Action = actionRunner(cmd, action)
})
app.Command("load", "Load a table dump from S3 or file to a DynamoDB table", func(cmd *cli.Cmd) {
cmd.Spec = "[-mpw] [--allow-overwrite] (--filename | --stdin | (--s3-bucket --s3-prefix)) TABLENAME"
action := &loader{
tableName: cmd.StringArg("TABLENAME", "", "Table name to load into"),
allowOverwrite: cmd.BoolOpt("allow-overwrite", false, "Set to true to overwrite any existing rows"),
filename: cmd.StringOpt("f filename", "", "Filename to read data from. Set to \"-\" for stdin"),
stdin: cmd.BoolOpt("stdin", false, "If true then read the dump data from stdin"),
maxItems: cmd.IntOpt("m maxitems", 0, "Maximum number of items to load. Set to 0 to process all items"),
parallel: cmd.IntOpt("p parallel", 4, "Number of concurrent channels to open to DynamoDB"),
writeCapacity: cmd.IntOpt("w write-capacity", 5, "Average aggregate write capacity to use for load (set to 0 for unlimited)"),
s3BucketName: cmd.StringOpt("s3-bucket", "", "S3 bucket name to read from"),
s3Prefix: cmd.StringOpt("s3-prefix", "", `Path prefix to use to read data from S3 (eg. "backups/2016-04-01-12:25-")`),
}
cmd.Before = func() {
checkGTE(*action.parallel, 1, "--parallel")
checkLTE(*action.parallel, maxParallel, "--parallel")
checkGTE(*action.maxItems, 0, "--max-items")
checkGTE(*action.writeCapacity, 0, "--write-capacity")
}
cmd.Action = actionRunner(cmd, action)
})
app.Command("info", "Display backup metadata from an S3 backup", func(cmd *cli.Cmd) {
cmd.Spec = "--s3-bucket --s3-prefix"
action := &metadataDumper{
s3BucketName: cmd.StringOpt("s3-bucket", "", "S3 bucket name to read from"),
s3Prefix: cmd.StringOpt("s3-prefix", "", `Path prefix to use to read data from S3 (eg. "backups/2016-04-01-12:25-")`),
}
cmd.Action = action.run
})
app.Command("delete", "Delete a backup from S3", func(cmd *cli.Cmd) {
cmd.Spec = "--s3-bucket --s3-prefix [--force]"
action := &deleter{
s3BucketName: cmd.StringOpt("s3-bucket", "", "S3 bucket name to delete from"),
s3Prefix: cmd.StringOpt("s3-prefix", "", `Path prefix to use to delete from S3 (eg. "backups/2016-04-01-12:25-")`),
force: cmd.BoolOpt("force", false, "Set to true to disable the delete prompt"),
}
cmd.Action = actionRunner(cmd, action)
})
app.Run(os.Args)
}