Switch to non-strict schema

Fixes quickwit-oss#1211
PSeitz · Nov 25, 2021 · 5429ea3 · 5429ea3
1 parent 02174d2
commit 5429ea3
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 29 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+Tantivy 0.17
+================================
+- Change to non-strict schema. Ignore fields in data which are not defined in schema. Previously this returned an error. #1211
+
 Tantivy 0.16.2
 ================================
 - Bugfix in FuzzyTermQuery. (tranposition_cost_one was not doing anything)

diff --git a/src/schema/schema.rs b/src/schema/schema.rs
@@ -271,8 +271,6 @@ impl Schema {
                     let field_value = FieldValue::new(field, value);
                     document.add(field_value);
                 }
-            } else {
-                return Err(DocParsingError::NoSuchFieldInSchema(field_name));
             }
         }
         Ok(document)
@@ -314,26 +312,25 @@ impl Schema {
 
         let mut doc = Document::default();
         for (field_name, json_value) in json_obj.iter() {
-            let field = self
-                .get_field(field_name)
-                .ok_or_else(|| DocParsingError::NoSuchFieldInSchema(field_name.clone()))?;
-            let field_entry = self.get_field_entry(field);
-            let field_type = field_entry.field_type();
-            match *json_value {
-                JsonValue::Array(ref json_items) => {
-                    for json_item in json_items {
+            if let Some(field) = self.get_field(field_name) {
+                let field_entry = self.get_field_entry(field);
+                let field_type = field_entry.field_type();
+                match *json_value {
+                    JsonValue::Array(ref json_items) => {
+                        for json_item in json_items {
+                            let value = field_type
+                                .value_from_json(json_item)
+                                .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
+                            doc.add(FieldValue::new(field, value));
+                        }
+                    }
+                    _ => {
                         let value = field_type
-                            .value_from_json(json_item)
+                            .value_from_json(json_value)
                             .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
                         doc.add(FieldValue::new(field, value));
                     }
                 }
-                _ => {
-                    let value = field_type
-                        .value_from_json(json_value)
-                        .map_err(|e| DocParsingError::ValueError(field_name.clone(), e))?;
-                    doc.add(FieldValue::new(field, value));
-                }
             }
         }
         Ok(doc)
@@ -398,9 +395,6 @@ pub enum DocParsingError {
     /// One of the value node could not be parsed.
     #[error("The field '{0:?}' could not be parsed: {1:?}")]
     ValueError(String, ValueParsingError),
-    /// The json-document contains a field that is not declared in the schema.
-    #[error("The document contains a field that is not declared in the schema: {0:?}")]
-    NoSuchFieldInSchema(String),
 }
 
 #[cfg(test)]
@@ -578,20 +572,16 @@ mod tests {
     }
 
     #[test]
-    pub fn test_document_from_nameddoc_error() {
+    pub fn test_document_missing_field_no_error() {
         let schema = Schema::builder().build();
         let mut named_doc_map = BTreeMap::default();
         named_doc_map.insert(
             "title".to_string(),
             vec![Value::from("title1"), Value::from("title2")],
         );
-        let err = schema
+        schema
             .convert_named_doc(NamedFieldDocument(named_doc_map))
-            .unwrap_err();
-        assert_eq!(
-            err,
-            DocParsingError::NoSuchFieldInSchema("title".to_string())
-        );
+            .unwrap();
     }
 
     #[test]
@@ -644,8 +634,9 @@ mod tests {
             );
         }
         {
-            let json_err = schema.parse_document(
+            let res = schema.parse_document(
                 r#"{
+                "thisfieldisnotdefinedintheschema": "my title",
                 "title": "my title",
                 "author": "fulmicoton",
                 "count": 4,
@@ -654,7 +645,7 @@ mod tests {
                 "jambon": "bayonne"
             }"#,
             );
-            assert_matches!(json_err, Err(DocParsingError::NoSuchFieldInSchema(_)));
+            assert!(res.is_ok());
         }
         {
             let json_err = schema.parse_document(