-
Notifications
You must be signed in to change notification settings - Fork 73
/
http_io.h
160 lines (145 loc) · 6.71 KB
/
http_io.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
/*
* s3backer - FUSE-based single file backing store via Amazon S3
*
* Copyright 2008-2023 Archie L. Cobbs <archie.cobbs@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the
* OpenSSL library under certain conditions as described in each
* individual source file, and distribute linked combinations including
* the two.
*
* You must obey the GNU General Public License in all respects for all
* of the code used other than OpenSSL. If you modify file(s) with this
* exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do
* so, delete this exception statement from your version. If you delete
* this exception statement from all source files in the program, then
* also delete it here.
*/
// Upload/download indexes
#define HTTP_DOWNLOAD 0
#define HTTP_UPLOAD 1
// Authentication types
#define AUTH_VERSION_AWS2 "aws2"
#define AUTH_VERSION_AWS4 "aws4"
// Storage classes
#define STORAGE_CLASS_STANDARD "STANDARD"
#define STORAGE_CLASS_STANDARD_IA "STANDARD_IA"
#define STORAGE_CLASS_ONEZONE_IA "ONEZONE_IA"
#define STORAGE_CLASS_REDUCED_REDUNDANCY "REDUCED_REDUNDANCY"
#define STORAGE_CLASS_INTELLIGENT_TIERING "INTELLIGENT_TIERING"
#define STORAGE_CLASS_GLACIER "GLACIER"
#define STORAGE_CLASS_DEEP_ARCHIVE "DEEP_ARCHIVE"
#define STORAGE_CLASS_OUTPOSTS "OUTPOSTS"
// Server side encryption types
#define SSE_AES256 "AES256"
#define SSE_AWS_KMS "aws:kms"
// Configuration info structure for http_io store
struct http_io_conf {
char *accessId;
char *accessKey;
char *iam_token;
const char *accessType;
const char *ec2iam_role;
const char *storage_class;
const char *authVersion;
const char *baseURL;
const char *vhostURL; // "baseURL" in --vhost format
const char *region;
const char *bucket;
const char *prefix;
const char *user_agent;
const char *cacert;
const char *password;
const char *encryption;
const char *default_ce;
u_int key_length;
int debug;
int debug_http;
int http_11; // restrict to HTTP 1.1
int quiet;
int no_curl_cache; // don't cache cURL handles
const struct comp_alg *compress_alg; // compression algorithm, or NULL for none
void *compress_level; // compression level info
int vhost; // use virtual host style URL
bitmap_t *nonzero_bitmap; // is set to NULL by http_io_create()
int blockHashPrefix;
int insecure;
u_int block_size;
s3b_block_t num_blocks;
int list_blocks_threads;
u_int timeout;
u_int initial_retry_pause;
u_int max_retry_pause;
uintmax_t max_speed[2];
log_func_t *log;
const char *sse;
const char *sse_key_id;
};
// Statistics structure for http_io store
struct http_io_evst {
u_int count; // number of occurrences
double time; // total time taken
};
struct http_io_stats {
// Block stats
u_int normal_blocks_read;
u_int normal_blocks_written;
u_int zero_blocks_read;
u_int zero_blocks_written;
u_int empty_blocks_read; // only when nonzero_bitmap != NULL
u_int empty_blocks_written; // only when nonzero_bitmap != NULL
// HTTP transfer stats
struct http_io_evst http_heads; // total successful
struct http_io_evst http_gets; // total successful
struct http_io_evst http_puts; // total successful
struct http_io_evst http_deletes; // total successful
u_int http_unauthorized;
u_int http_forbidden;
u_int http_stale;
u_int http_redirect;
u_int http_verified;
u_int http_mismatch;
u_int http_5xx_error;
u_int http_4xx_error;
u_int http_3xx_error;
u_int http_other_error;
u_int http_canceled_writes;
// CURL stats
u_int curl_handles_created;
u_int curl_handles_reused;
u_int curl_timeouts;
u_int curl_connect_failed;
u_int curl_host_unknown;
u_int curl_out_of_memory;
u_int curl_other_error;
// Retry stats
u_int num_retries;
uint64_t retry_delay;
// Misc
u_int out_of_memory_errors;
};
// http_io.c
extern struct s3backer_store *http_io_create(struct http_io_conf *config);
extern void http_io_get_stats(struct s3backer_store *s3b, struct http_io_stats *stats);
extern void http_io_clear_stats(struct s3backer_store *s3b);
extern int http_io_parse_block(const char *prefix, s3b_block_t num_blocks,
int blockHashPrefix, const char *name, s3b_block_t *hash_valuep, s3b_block_t *block_nump);
extern void http_io_format_block_hash(int blockHashPrefix, char *block_hash_buf, size_t bufsiz, s3b_block_t block_num);
extern int http_io_list_blocks(struct s3backer_store *s3b, block_list_func_t *callback, void *arg);