From ed49b0067e82780f46f44e47abea68d9134527a8 Mon Sep 17 00:00:00 2001 From: Thijs Cadier Date: Fri, 3 Nov 2017 14:26:51 +0100 Subject: [PATCH 1/7] Bump to 0.11, use pkg version in build of sys package --- CHANGELOG.md | 4 ++++ Cargo.toml | 4 ++-- mongoc-sys/Cargo.toml | 2 +- mongoc-sys/build.rs | 18 +++++++++++------- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e15c9b..7ee6900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.11.0 +* Update bson dependency to 0.10 +* Use installed libmongoc if it right version is present on the system (thanks to Matrix-Zhang) + # 0.10.0 * Initial upgrade to mongo c driver 1.8.0, no support for new features yet diff --git a/Cargo.toml b/Cargo.toml index 9616294..59b9f92 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mongo_driver" -version = "0.10.0" +version = "0.11.0" authors = ["Thijs Cadier "] description = "Mongo Rust driver built on top of the Mongo C driver" readme = "README.md" @@ -25,7 +25,7 @@ bson = "^0.10" [dependencies.mongoc-sys] path = "mongoc-sys" -version = "1.8.0" +version = "1.8.0-0" [dev-dependencies] chrono = "^0.4" diff --git a/mongoc-sys/Cargo.toml b/mongoc-sys/Cargo.toml index c28f6fa..3e0d6da 100644 --- a/mongoc-sys/Cargo.toml +++ b/mongoc-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mongoc-sys" -version = "1.8.0" +version = "1.8.0-0" description = "Sys package with installer and bindings for mongoc" authors = ["Thijs Cadier "] build = "build.rs" diff --git a/mongoc-sys/build.rs b/mongoc-sys/build.rs index c11dfda..a57ccae 100644 --- a/mongoc-sys/build.rs +++ b/mongoc-sys/build.rs @@ -4,26 +4,30 @@ use std::env; use std::path::Path; use std::process::Command; -static VERSION: &'static str = "1.8.0"; // Should be the same major version as in the manifest fn main() { + let mongoc_version = env!("CARGO_PKG_VERSION") + .split('-') + .next() + .expect("Crate version is not valid"); + if pkg_config::Config::new() - .atleast_version(VERSION) + .atleast_version(mongoc_version) .statik(true) .probe("libmongoc-1.0") .is_err() { let out_dir_var = env::var("OUT_DIR").expect("No out dir"); - let out_dir = format!("{}/{}", out_dir_var, VERSION); - let driver_src_path = format!("mongo-c-driver-{}", VERSION); + let out_dir = format!("{}/{}", out_dir_var, mongoc_version); + let driver_src_path = format!("mongo-c-driver-{}", mongoc_version); let libmongoc_path = Path::new(&out_dir).join("lib/libmongoc-1.0.a"); if !libmongoc_path.exists() { // Download and extract driver archive let url = format!( "https://github.com/mongodb/mongo-c-driver/releases/download/{}/mongo-c-driver-{}.tar.gz", - VERSION, - VERSION + mongoc_version, + mongoc_version ); assert!( Command::new("curl").arg("-O") // Save to disk @@ -34,7 +38,7 @@ fn main() { .success() ); - let archive_name = format!("mongo-c-driver-{}.tar.gz", VERSION); + let archive_name = format!("mongo-c-driver-{}.tar.gz", mongoc_version); assert!( Command::new("tar") .arg("xzf") From 2edd61d2e9a32913b63af390836b380d7d302ed0 Mon Sep 17 00:00:00 2001 From: Thijs Cadier Date: Sun, 31 Dec 2017 07:53:02 +0100 Subject: [PATCH 2/7] Add test for invalid utf8 in bulk operation --- tests/bulk_operation.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/bulk_operation.rs b/tests/bulk_operation.rs index d6acabe..797b7a4 100644 --- a/tests/bulk_operation.rs +++ b/tests/bulk_operation.rs @@ -57,6 +57,26 @@ fn test_utf8() { ); } +#[test] +fn test_utf8_invalid() { + let uri = Uri::new("mongodb://localhost:27017/").unwrap(); + let pool = ClientPool::new(uri, None); + let client = pool.pop(); + let collection = client.get_collection("rust_driver_test", "bulk_operation_utf8_invalid"); + let bulk_operation = collection.create_bulk_operation(None); + + let value = unsafe { String::from_utf8_unchecked(b"\xF0\xA4\xAD\xA2".to_vec()) }; + let document = doc! {"key_1" => value}; + bulk_operation.insert(&document).unwrap(); + assert!(bulk_operation.execute().is_ok()); + + let first_document = collection.find(&doc!{}, None).unwrap().next().unwrap().unwrap(); + assert_eq!( + first_document.get("key_1").unwrap(), + &bson::Bson::String("𤭢".to_string()) + ); +} + #[test] fn test_insert_remove_replace_update_extended() { if env::var("SKIP_EXTENDED_BULK_OPERATION_TESTS") == Ok("true".to_string()) { From 2eae40d9c6c751e751e303a40890449a363c900b Mon Sep 17 00:00:00 2001 From: Thijs Cadier Date: Sun, 31 Dec 2017 07:54:52 +0100 Subject: [PATCH 3/7] Better utf8 invalid test --- tests/bulk_operation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bulk_operation.rs b/tests/bulk_operation.rs index 797b7a4..918d334 100644 --- a/tests/bulk_operation.rs +++ b/tests/bulk_operation.rs @@ -65,7 +65,7 @@ fn test_utf8_invalid() { let collection = client.get_collection("rust_driver_test", "bulk_operation_utf8_invalid"); let bulk_operation = collection.create_bulk_operation(None); - let value = unsafe { String::from_utf8_unchecked(b"\xF0\xA4\xAD\xA2".to_vec()) }; + let value = unsafe { String::from_utf8_unchecked(vec![10, 11, 12, 13, 14, 15, 16, 17]) }; let document = doc! {"key_1" => value}; bulk_operation.insert(&document).unwrap(); assert!(bulk_operation.execute().is_ok()); From bf23a5d470ff7ed2be24e668bd2685e9ab9d3771 Mon Sep 17 00:00:00 2001 From: Thijs Cadier Date: Sat, 6 Jan 2018 07:30:13 +0100 Subject: [PATCH 4/7] Bump libmongoc to 1.8.2 --- Cargo.toml | 2 +- mongoc-sys/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 59b9f92..67fa75e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ bson = "^0.10" [dependencies.mongoc-sys] path = "mongoc-sys" -version = "1.8.0-0" +version = "1.8.2-0" [dev-dependencies] chrono = "^0.4" diff --git a/mongoc-sys/Cargo.toml b/mongoc-sys/Cargo.toml index 3e0d6da..4a04fe4 100644 --- a/mongoc-sys/Cargo.toml +++ b/mongoc-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mongoc-sys" -version = "1.8.0-0" +version = "1.8.2-0" description = "Sys package with installer and bindings for mongoc" authors = ["Thijs Cadier "] build = "build.rs" From 8e8a02f4540ca04f56efaba3fd1fa979558ea157 Mon Sep 17 00:00:00 2001 From: Thijs Cadier Date: Sat, 6 Jan 2018 07:38:43 +0100 Subject: [PATCH 5/7] Different invalid utf-8 test string --- tests/bulk_operation.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/bulk_operation.rs b/tests/bulk_operation.rs index 918d334..a31b456 100644 --- a/tests/bulk_operation.rs +++ b/tests/bulk_operation.rs @@ -65,7 +65,8 @@ fn test_utf8_invalid() { let collection = client.get_collection("rust_driver_test", "bulk_operation_utf8_invalid"); let bulk_operation = collection.create_bulk_operation(None); - let value = unsafe { String::from_utf8_unchecked(vec![10, 11, 12, 13, 14, 15, 16, 17]) }; + let bytes = b"\x80\xae".to_vec(); + let value = unsafe { String::from_utf8_unchecked(bytes) }; let document = doc! {"key_1" => value}; bulk_operation.insert(&document).unwrap(); assert!(bulk_operation.execute().is_ok()); From d0b93b6bb06b642953972f067d29797c11530ebd Mon Sep 17 00:00:00 2001 From: Thijs Cadier Date: Tue, 16 Jan 2018 12:14:48 +0100 Subject: [PATCH 6/7] Bump bson to 0.11 and use lossy utf-8 decoding for responses Mongo can return invalid UTF-8 in bson responses: https://jira.mongodb.org/browse/SERVER-24007 We use lossy UTF-8 decoding when parsing now. --- Cargo.toml | 2 +- src/bsonc.rs | 36 +++++++++++++++++++++++++++++++++++- src/client.rs | 2 +- src/collection.rs | 6 +++--- src/database.rs | 2 +- tests/bulk_operation.rs | 37 +++++++++++-------------------------- 6 files changed, 52 insertions(+), 33 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 67fa75e..1a0fa27 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ name = "tests" [dependencies] libc = "^0.2" log = "^0.3" -bson = "^0.10" +bson = "^0.11" [dependencies.mongoc-sys] path = "mongoc-sys" diff --git a/src/bsonc.rs b/src/bsonc.rs index c033141..45bf2d6 100644 --- a/src/bsonc.rs +++ b/src/bsonc.rs @@ -42,6 +42,7 @@ impl Bsonc { Ok(Bsonc{ inner: inner }) } + /// Decode a bson from the C side to a document pub fn as_document(&self) -> Result { assert!(!self.inner.is_null()); @@ -63,6 +64,28 @@ impl Bsonc { Ok(document) } + /// Decode a bson from the C side to a document with lossy UTF-8 decoding + pub fn as_document_utf8_lossy(&self) -> Result { + assert!(!self.inner.is_null()); + + // This pointer should not be modified or freed + // See: http://mongoc.org/libbson/current/bson_get_data.html + let data_ptr = unsafe { bindings::bson_get_data(self.inner) }; + assert!(!data_ptr.is_null()); + + let data_len = unsafe { + let bson = *self.inner; + bson.len + } as usize; + + let mut slice = unsafe { + slice::from_raw_parts(data_ptr, data_len) + }; + + let document = try!(bson::decode_document_utf8_lossy(&mut slice)); + Ok(document) + } + pub fn as_json(&self) -> String { assert!(!self.inner.is_null()); let json_ptr = unsafe { bindings::bson_as_json(self.inner, ptr::null_mut()) }; @@ -106,7 +129,18 @@ mod tests { let bsonc = super::Bsonc::from_document(&document).unwrap(); let decoded = bsonc.as_document().unwrap(); - assert!(decoded.contains_key("key")); + assert_eq!(decoded.get_str("key").unwrap(), "value"); + } + + #[test] + fn test_bsonc_from_and_as_document_invalid_utf8() { + let bytes = b"\x80\xae".to_vec(); + let value = unsafe { String::from_utf8_unchecked(bytes) }; + let document = doc! { "key" => value }; + let bsonc = super::Bsonc::from_document(&document).unwrap(); + + let decoded = bsonc.as_document_utf8_lossy().unwrap(); + assert_eq!(decoded.get_str("key").unwrap(), "��"); } #[test] diff --git a/src/client.rs b/src/client.rs index 3d65629..2fe78aa 100644 --- a/src/client.rs +++ b/src/client.rs @@ -305,7 +305,7 @@ impl<'a> Client<'a> { }; if success == 1 { - match reply.as_document() { + match reply.as_document_utf8_lossy() { Ok(document) => return Ok(document), Err(error) => return Err(error.into()) } diff --git a/src/collection.rs b/src/collection.rs index 72a50df..729551f 100644 --- a/src/collection.rs +++ b/src/collection.rs @@ -342,7 +342,7 @@ impl<'a> Collection<'a> { }; if success == 1 { - match reply.as_document() { + match reply.as_document_utf8_lossy() { Ok(document) => return Ok(document), Err(error) => return Err(error.into()) } @@ -547,7 +547,7 @@ impl<'a> Collection<'a> { }; if success == 1 { - match reply.as_document() { + match reply.as_document_utf8_lossy() { Ok(document) => return Ok(document), Err(error) => return Err(error.into()) } @@ -884,7 +884,7 @@ impl<'a>BulkOperation<'a> { ) }; - let document = match reply.as_document() { + let document = match reply.as_document_utf8_lossy() { Ok(document) => document, Err(error) => return Err(BulkOperationError{error: error.into(), reply: doc!{}}) }; diff --git a/src/database.rs b/src/database.rs index bf97f31..6cb05af 100644 --- a/src/database.rs +++ b/src/database.rs @@ -116,7 +116,7 @@ impl<'a> Database<'a> { }; if success == 1 { - match reply.as_document() { + match reply.as_document_utf8_lossy() { Ok(document) => return Ok(document), Err(error) => return Err(error.into()) } diff --git a/tests/bulk_operation.rs b/tests/bulk_operation.rs index a31b456..7a8c115 100644 --- a/tests/bulk_operation.rs +++ b/tests/bulk_operation.rs @@ -9,7 +9,9 @@ fn test_execute_error() { let uri = Uri::new("mongodb://localhost:27017/").unwrap(); let pool = ClientPool::new(uri, None); let client = pool.pop(); - let collection = client.get_collection("rust_driver_test", "bulk_operation_error"); + let mut collection = client.get_collection("rust_driver_test", "bulk_operation_error"); + collection.drop().unwrap_or(()); + let bulk_operation = collection.create_bulk_operation(None); let result = bulk_operation.execute(); @@ -24,11 +26,13 @@ fn test_basics() { let uri = Uri::new("mongodb://localhost:27017/").unwrap(); let pool = ClientPool::new(uri, None); let client = pool.pop(); - let collection = client.get_collection("rust_driver_test", "bulk_operation_basics"); + let mut collection = client.get_collection("rust_driver_test", "bulk_operation_basics"); + collection.drop().unwrap_or(()); + let bulk_operation = collection.create_bulk_operation(None); let document = doc! {"key_1" => "Value 1"}; - bulk_operation.insert(&document).unwrap(); + bulk_operation.insert(&document).expect("Could not insert"); assert!(bulk_operation.execute().is_ok()); let first_document = collection.find(&doc!{}, None).unwrap().next().unwrap().unwrap(); @@ -43,11 +47,13 @@ fn test_utf8() { let uri = Uri::new("mongodb://localhost:27017/").unwrap(); let pool = ClientPool::new(uri, None); let client = pool.pop(); - let collection = client.get_collection("rust_driver_test", "bulk_operation_utf8"); + let mut collection = client.get_collection("rust_driver_test", "bulk_operation_utf8"); + collection.drop().unwrap_or(()); + let bulk_operation = collection.create_bulk_operation(None); let document = doc! {"key_1" => "kācaṃ śaknomyattum; nopahinasti mām."}; - bulk_operation.insert(&document).unwrap(); + bulk_operation.insert(&document).expect("Could not insert"); assert!(bulk_operation.execute().is_ok()); let first_document = collection.find(&doc!{}, None).unwrap().next().unwrap().unwrap(); @@ -57,27 +63,6 @@ fn test_utf8() { ); } -#[test] -fn test_utf8_invalid() { - let uri = Uri::new("mongodb://localhost:27017/").unwrap(); - let pool = ClientPool::new(uri, None); - let client = pool.pop(); - let collection = client.get_collection("rust_driver_test", "bulk_operation_utf8_invalid"); - let bulk_operation = collection.create_bulk_operation(None); - - let bytes = b"\x80\xae".to_vec(); - let value = unsafe { String::from_utf8_unchecked(bytes) }; - let document = doc! {"key_1" => value}; - bulk_operation.insert(&document).unwrap(); - assert!(bulk_operation.execute().is_ok()); - - let first_document = collection.find(&doc!{}, None).unwrap().next().unwrap().unwrap(); - assert_eq!( - first_document.get("key_1").unwrap(), - &bson::Bson::String("𤭢".to_string()) - ); -} - #[test] fn test_insert_remove_replace_update_extended() { if env::var("SKIP_EXTENDED_BULK_OPERATION_TESTS") == Ok("true".to_string()) { From 50b10ad52e68cde6038aa83eb22313950f65b11b Mon Sep 17 00:00:00 2001 From: Thijs Cadier Date: Tue, 16 Jan 2018 13:57:09 +0100 Subject: [PATCH 7/7] Bump to 0.12 --- CHANGELOG.md | 6 ++++++ Cargo.toml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ee6900..f415552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 0.12.0 +* Upgrade libmongoc to 1.8.2 +* Update bson dependency to 0.11 +* Parse MongoDB server responses with lossy UTF-8 decoding to work + around https://jira.mongodb.org/browse/SERVER-24007 + # 0.11.0 * Update bson dependency to 0.10 * Use installed libmongoc if it right version is present on the system (thanks to Matrix-Zhang) diff --git a/Cargo.toml b/Cargo.toml index 1a0fa27..99fb54b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mongo_driver" -version = "0.11.0" +version = "0.12.0" authors = ["Thijs Cadier "] description = "Mongo Rust driver built on top of the Mongo C driver" readme = "README.md"