Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update flatbuffers to 22.9(resolved RUSTSEC-2021-0122) #794

Merged
merged 2 commits into from
Dec 8, 2022
Merged

Update flatbuffers to 22.9(resolved RUSTSEC-2021-0122) #794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3f34be5
fix: update flatbuffers to 22.9(resolved resolve RUSTSEC-2021-0122)
zipper-meng Dec 8, 2022
72fb943
fix: upgrade flatbuffers version in CI scripts
zipper-meng Dec 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 1 addition & 2 deletions .github/actions/setup-builder/action.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ runs:
- name: Install Build Dependencies
shell: bash
run: |
git clone https://github.com/google/flatbuffers.git
git clone -b v22.9.29 --depth 1 https://github.com/google/flatbuffers.git
cd flatbuffers
git checkout v2.0.6
cmake .
make
sudo make install
Expand Down
7 changes: 3 additions & 4 deletions .github/workflows/makefile.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,17 @@ jobs:

steps:
- uses: actions/checkout@v3

- name: dep
run: |
git clone https://github.com/google/flatbuffers.git
git clone -b v22.9.29 --depth 1 https://github.com/google/flatbuffers.git
cd flatbuffers
git checkout v2.0.6
cmake .
make
sudo make install
sudo ldconfig
flatc --version

- name: test
run: make test

Expand Down
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ diff = "0.1.13"
dirs = "4.0.0"
env_logger = "0.9"
evmap = "10.0"
flatbuffers = "2.1"
flatbuffers = "22.9"
flate2 = "1.0.24"
futures = { version = "0.3" }
integer-encoding = "3.0.3"
Expand Down
1 change: 1 addition & 0 deletions common/models/src/field_info.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ impl FieldInfo {
.ok_or(Error::InvalidFlatbufferMessage {
err: "".to_string(),
})?
.bytes()
.to_vec(),
value_type: field.type_().into(),
code_type: 0,
Expand Down
4 changes: 2 additions & 2 deletions common/models/src/points.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ impl FieldValue {
field_id: 0,
value_type: field.type_().into(),
value: match field.value() {
Some(v) => v.to_vec(),
Some(v) => v.bytes().to_vec(),
None => Vec::new(),
},
})
Expand Down Expand Up @@ -72,7 +72,7 @@ impl From<fb_models::Point<'_>> for InMemPoint {
let mut fields = Vec::with_capacity(fields_inner.len());
for f in fields_inner.into_iter() {
let val_type = f.type_().into();
let val = f.value().unwrap().to_vec();
let val = f.value().unwrap().bytes().to_vec();
fields.push(FieldValue {
field_id: 0,
value_type: val_type,
Expand Down
16 changes: 9 additions & 7 deletions common/models/src/series_info.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,11 @@ impl SeriesKey {
};

let db = match point.db() {
Some(db) => String::from_utf8(db.to_vec()).map_err(|err| InvalidFlatbufferMessage {
err: err.to_string(),
})?,
Some(db) => {
String::from_utf8(db.bytes().to_vec()).map_err(|err| InvalidFlatbufferMessage {
err: err.to_string(),
})?
}

None => {
return Err(Error::InvalidFlatbufferMessage {
Expand All @@ -104,11 +106,11 @@ impl SeriesKey {
};

let table = match point.tab() {
Some(table) => {
String::from_utf8(table.to_vec()).map_err(|err| InvalidFlatbufferMessage {
Some(table) => String::from_utf8(table.bytes().to_vec()).map_err(|err| {
InvalidFlatbufferMessage {
err: err.to_string(),
})?
}
}
})?,

None => {
return Err(InvalidFlatbufferMessage {
Expand Down
2 changes: 2 additions & 0 deletions common/models/src/tag.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,14 @@ impl Tag {
.ok_or(Error::InvalidFlatbufferMessage {
err: "Tag key cannot be empty".to_string(),
})?
.bytes()
.to_vec();
let value = tag
.value()
.ok_or(Error::InvalidFlatbufferMessage {
err: "Tag value cannot be empty".to_string(),
})?
.bytes()
.to_vec();
Ok(Self { key, value })
}
Expand Down
18 changes: 14 additions & 4 deletions common/protos/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,17 @@ mod flatbuffers_generated;
flatbuffers_generated_mod_rs_file.write_all(b";\n")?;
flatbuffers_generated_mod_rs_file.flush()?;

let output = Command::new("flatc")
let flatc_path = match env::var("FLATC_PATH") {
Ok(p) => {
println!(
"Found specified flatc path in environment FLATC_PATH( {} )",
&p
);
p
}
Err(_) => "flatc".to_string(),
};
let output = Command::new(&flatc_path)
.arg("-o")
.arg(&output_dir_final)
.arg("--rust")
Expand All @@ -109,10 +119,10 @@ mod flatbuffers_generated;
.arg("")
.arg(p)
.output()
.unwrap_or_else(|_| {
.unwrap_or_else(|e| {
panic!(
"Failed to generate file by flatbuffers {}.",
output_file_name
"Failed to generate file '{}' by flatc(path: '{}'): {:?}.",
output_file_name, flatc_path, e
)
});

Expand Down
2 changes: 1 addition & 1 deletion docs/quick-start-cn.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@
如果您的系统不在此列，需要自行编译

```shell
$ git clone -b v2.0.6 --depth 1 https://github.com/google/flatbuffers.git && cd flatbuffers
$ git clone -b v22.9.29 --depth 1 https://github.com/google/flatbuffers.git && cd flatbuffers

# 根据操作系统选择以下命令之一
$ cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release
Expand Down
3 changes: 2 additions & 1 deletion docs/quick-start.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,15 @@ We support the following platforms, please report to us if you find it works on
If your system is not listed here, you can install FlatBuffers as follows

```shell
git clone -b v2.0.6 --depth 1 https://github.com/google/flatbuffers.git && cd flatbuffers
git clone -b v22.9.29 --depth 1 https://github.com/google/flatbuffers.git && cd flatbuffers
```

```shell
# Choose one of the following commands depending on the operating system
cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release
cmake -G "Visual Studio 10" -DCMAKE_BUILD_TYPE=Release
cmake -G "Xcode" -DCMAKE_BUILD_TYPE=Release

sudo make install
```

Expand Down
2 changes: 1 addition & 1 deletion tskv/src/database.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ impl Database {
point: Point,
sid: u64,
) -> Result<()> {
let table_name = String::from_utf8(point.tab().unwrap().to_vec()).unwrap();
let table_name = String::from_utf8(point.tab().unwrap().bytes().to_vec()).unwrap();
let table_schema = self
.index
.get_table_schema(&table_name)
Expand Down
19 changes: 12 additions & 7 deletions tskv/src/index/db_index.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,8 @@ impl DBIndex {
}

pub fn check_field_type_from_cache(&self, series_id: u64, info: &Point) -> IndexResult<()> {
let table_name = unsafe { String::from_utf8_unchecked(info.tab().unwrap().to_vec()) };
let table_name =
unsafe { String::from_utf8_unchecked(info.tab().unwrap().bytes().to_vec()) };
if let Some(schema) = self.table_schema.read().get(&table_name) {
let schema = match schema {
TableSchema::TsKvTableSchema(schema) => schema,
Expand All @@ -218,7 +219,7 @@ impl DBIndex {
}
};
for field in info.fields().unwrap() {
let field_name = String::from_utf8(field.name().unwrap().to_vec()).unwrap();
let field_name = String::from_utf8(field.name().unwrap().bytes().to_vec()).unwrap();
if let Some(v) = schema.column(&field_name) {
if field.type_().0 != v.column_type.field_type() as i32 {
trace::debug!(
Expand All @@ -239,7 +240,8 @@ impl DBIndex {
}
}
for tag in info.tags().unwrap() {
let tag_name: String = String::from_utf8(tag.key().unwrap().to_vec()).unwrap();
let tag_name: String =
String::from_utf8(tag.key().unwrap().bytes().to_vec()).unwrap();
if let Some(v) = schema.column(&tag_name) {
if ColumnType::Tag != v.column_type {
trace::debug!("type mismatch, point: tag, schema: {}", &v.column_type);
Expand All @@ -260,8 +262,9 @@ impl DBIndex {
pub fn check_field_type_or_else_add(&self, series_id: u64, info: &Point) -> IndexResult<()> {
//load schema first from cache,or else from storage and than cache it!
let mut schema = &mut TskvTableSchema::default();
let table_name = unsafe { String::from_utf8_unchecked(info.tab().unwrap().to_vec()) };
let db_name = unsafe { String::from_utf8_unchecked(info.db().unwrap().to_vec()) };
let table_name =
unsafe { String::from_utf8_unchecked(info.tab().unwrap().bytes().to_vec()) };
let db_name = unsafe { String::from_utf8_unchecked(info.db().unwrap().bytes().to_vec()) };
let mut fields = self.table_schema.write();
let mut new_schema = false;
match fields.get_mut(&table_name) {
Expand Down Expand Up @@ -333,13 +336,15 @@ impl DBIndex {

//check tags
for tag in info.tags().unwrap() {
let tag_key = unsafe { String::from_utf8_unchecked(tag.key().unwrap().to_vec()) };
let tag_key =
unsafe { String::from_utf8_unchecked(tag.key().unwrap().bytes().to_vec()) };
check_fn(&mut TableColumn::new_with_default(tag_key, ColumnType::Tag))?
}

//check fields
for field in info.fields().unwrap() {
let field_name = unsafe { String::from_utf8_unchecked(field.name().unwrap().to_vec()) };
let field_name =
unsafe { String::from_utf8_unchecked(field.name().unwrap().bytes().to_vec()) };
check_fn(&mut TableColumn::new_with_default(
field_name,
ColumnType::from_i32(field.type_().0),
Expand Down
4 changes: 2 additions & 2 deletions tskv/src/kvcore.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,7 @@ impl Engine for TsKv {
let fb_points = flatbuffers::root::<fb_models::Points>(&points)
.context(error::InvalidFlatbufferSnafu)?;

let db_name = String::from_utf8(fb_points.db().unwrap().to_vec())
let db_name = String::from_utf8(fb_points.db().unwrap().bytes().to_vec())
.map_err(|err| Error::ErrCharacterSet)?;

let db_warp = self.version_set.read().get_db(&db_name);
Expand Down Expand Up @@ -421,7 +421,7 @@ impl Engine for TsKv {
let fb_points = flatbuffers::root::<fb_models::Points>(&points)
.context(error::InvalidFlatbufferSnafu)?;

let db_name = String::from_utf8(fb_points.db().unwrap().to_vec())
let db_name = String::from_utf8(fb_points.db().unwrap().bytes().to_vec())
.map_err(|err| Error::ErrCharacterSet)?;

let db = self
Expand Down
6 changes: 3 additions & 3 deletions tskv/src/memcache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,9 @@ impl RowData {
}
for (i, f) in fields_inner.into_iter().enumerate() {
let vtype = f.type_().into();
let val = MiniVec::from(f.value().unwrap());
let val = MiniVec::from(f.value().unwrap().bytes());
match schema.column(
String::from_utf8(f.name().unwrap().to_vec())
String::from_utf8(f.name().unwrap().bytes().to_vec())
.unwrap()
.as_str(),
) {
Expand Down Expand Up @@ -177,7 +177,7 @@ impl From<fb_models::Point<'_>> for RowData {
let mut fields = Vec::with_capacity(fields_inner.len());
for f in fields_inner.into_iter() {
let vtype = f.type_().into();
let val = MiniVec::from(f.value().unwrap());
let val = MiniVec::from(f.value().unwrap().bytes());
fields.push(Some(FieldVal::new(val, vtype)));
}
fields
Expand Down