diff --git a/README.md b/README.md index f393423eea..7ec875697d 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ Each `GET` method has a `PUT` companion `sync` and `async` methods are generic o | | | | --------------------------- | ------------------------------------------------------------------------------------------------- | | `async/sync/async-blocking` | [delete_object](https://docs.rs/rust-s3/latest/s3/bucket/struct.Bucket.html#method.delete_object) | +| `async/sync/async-blocking` | [delete_objects](https://docs.rs/rust-s3/latest/s3/bucket/struct.Bucket.html#method.delete_objects) | #### Location diff --git a/s3/src/bucket.rs b/s3/src/bucket.rs index 7cf34d861b..2566f609af 100644 --- a/s3/src/bucket.rs +++ b/s3/src/bucket.rs @@ -90,8 +90,9 @@ use crate::error::S3Error; use crate::post_policy::PresignedPost; use crate::serde_types::{ BucketLifecycleConfiguration, BucketLocationResult, CompleteMultipartUploadData, - CorsConfiguration, GetObjectAttributesOutput, HeadObjectResult, - InitiateMultipartUploadResponse, ListBucketResult, ListMultipartUploadsResult, Part, + CorsConfiguration, DeleteObjectsRequest, DeleteObjectsResult, GetObjectAttributesOutput, + HeadObjectResult, InitiateMultipartUploadResponse, ListBucketResult, + ListMultipartUploadsResult, ObjectIdentifier, Part, }; #[allow(unused_imports)] use crate::utils::{PutStreamResponse, error_from_response_data}; @@ -2116,6 +2117,92 @@ impl Bucket { request.response_data(false).await } + /// Delete multiple objects from S3 using the Multi-Object Delete API. + /// + /// If more than 1000 objects are provided, they are automatically batched + /// into multiple requests (S3 allows at most 1000 keys per request). + /// Results from all batches are combined into a single response. + /// + /// # Example: + /// + /// ```no_run + /// use s3::bucket::Bucket; + /// use s3::creds::Credentials; + /// use s3::serde_types::ObjectIdentifier; + /// use anyhow::Result; + /// + /// # #[tokio::main] + /// # async fn main() -> Result<()> { + /// + /// let bucket_name = "rust-s3-test"; + /// let region = "us-east-1".parse()?; + /// let credentials = Credentials::default()?; + /// let bucket = Bucket::new(bucket_name, region, credentials)?; + /// + /// let objects = vec![ + /// ObjectIdentifier::new("file1.txt"), + /// ObjectIdentifier::new("file2.txt"), + /// ObjectIdentifier::new("file3.txt"), + /// ]; + /// + /// // Async variant with `tokio` or `async-std` features + /// let response = bucket.delete_objects(objects).await?; + /// + /// // `sync` feature will produce an identical method + /// #[cfg(feature = "sync")] + /// let response = bucket.delete_objects(objects)?; + /// + /// // Blocking variant, generated with `blocking` feature in combination + /// // with `tokio` or `async-std` features. + /// #[cfg(feature = "blocking")] + /// let response = bucket.delete_objects_blocking(objects)?; + /// # + /// # Ok(()) + /// # } + /// ``` + #[maybe_async::maybe_async] + pub async fn delete_objects>>( + &self, + objects: I, + ) -> Result { + let objects = objects.into(); + let mut result = DeleteObjectsResult { + deleted: Vec::new(), + errors: Vec::new(), + }; + + // Strip leading '/' from keys to match library convention. + // Other methods (put_object, delete_object, etc.) strip the leading + // slash when building the URL; we do the same for the XML body. + let objects: Vec = objects + .into_iter() + .map(|mut obj| { + if let Some(stripped) = obj.key.strip_prefix('/') { + obj.key = stripped.to_string(); + } + obj + }) + .collect(); + + for chunk in objects.chunks(1000) { + let data = DeleteObjectsRequest { + objects: chunk.to_vec(), + quiet: false, + }; + let command = Command::DeleteObjects { data }; + let request = RequestImpl::new(self, "/", command).await?; + let response_data = request.response_data(false).await?; + if response_data.status_code() >= 300 { + return Err(error_from_response_data(response_data)?); + } + let msg: DeleteObjectsResult = quick_xml::de::from_str(response_data.as_str()?)?; + result.deleted.extend(msg.deleted); + result.errors.extend(msg.errors); + } + + Ok(result) + } + /// Head object from S3. /// /// # Example: @@ -3784,6 +3871,64 @@ mod test { put_head_delete_object_with_headers(*test_r2_bucket()).await; } + #[maybe_async::maybe_async] + async fn put_delete_objects(bucket: Bucket) { + use crate::serde_types::ObjectIdentifier; + + let paths = [ + "/+bulk_delete_1.file", + "/+bulk_delete_2.file", + "/+bulk_delete_3.file", + ]; + let test: Vec = object(128); + + // Put test objects + for path in &paths { + let response_data = bucket.put_object(*path, &test).await.unwrap(); + assert_eq!(response_data.status_code(), 200); + } + + // Bulk delete them + let objects: Vec = + paths.iter().map(|p| ObjectIdentifier::new(*p)).collect(); + let result = bucket.delete_objects(objects).await.unwrap(); + + assert_eq!(result.deleted.len(), 3); + assert!(result.errors.is_empty()); + + // Verify they are gone + for path in &paths { + let exists = bucket.object_exists(*path).await.unwrap(); + assert!(!exists); + } + } + + #[ignore] + #[maybe_async::test( + feature = "sync", + async(all(not(feature = "sync"), feature = "with-tokio"), tokio::test), + async( + all(not(feature = "sync"), feature = "with-async-std"), + async_std::test + ) + )] + async fn aws_test_delete_objects() { + put_delete_objects(*test_aws_bucket()).await; + } + + #[ignore] + #[maybe_async::test( + feature = "sync", + async(all(not(feature = "sync"), feature = "with-tokio"), tokio::test), + async( + all(not(feature = "sync"), feature = "with-async-std"), + async_std::test + ) + )] + async fn minio_test_delete_objects() { + put_delete_objects(*test_minio_bucket()).await; + } + #[maybe_async::test( feature = "sync", async(all(not(feature = "sync"), feature = "with-tokio"), tokio::test), diff --git a/s3/src/command.rs b/s3/src/command.rs index c56ca3ee46..4b57486af8 100644 --- a/s3/src/command.rs +++ b/s3/src/command.rs @@ -24,6 +24,7 @@ use std::collections::HashMap; use crate::error::S3Error; use crate::serde_types::{ BucketLifecycleConfiguration, CompleteMultipartUploadData, CorsConfiguration, + DeleteObjectsRequest, }; use crate::EMPTY_PAYLOAD_SHA; @@ -171,6 +172,9 @@ pub enum Command<'a> { expected_bucket_owner: String, version_id: Option, }, + DeleteObjects { + data: DeleteObjectsRequest, + }, } impl<'a> Command<'a> { @@ -203,9 +207,9 @@ impl<'a> Command<'a> { | Command::DeleteBucket | Command::DeleteBucketCors { .. } | Command::DeleteBucketLifecycle => HttpMethod::Delete, - Command::InitiateMultipartUpload { .. } | Command::CompleteMultipartUpload { .. } => { - HttpMethod::Post - } + Command::InitiateMultipartUpload { .. } + | Command::CompleteMultipartUpload { .. } + | Command::DeleteObjects { .. } => HttpMethod::Post, Command::HeadObject => HttpMethod::Head, Command::GetObjectAttributes { .. } => HttpMethod::Get, } @@ -252,6 +256,7 @@ impl<'a> Command<'a> { Command::GetBucketLifecycle => 0, Command::DeleteBucketLifecycle { .. } => 0, Command::GetObjectAttributes { .. } => 0, + Command::DeleteObjects { data } => data.len(), }; Ok(result) } @@ -289,6 +294,7 @@ impl<'a> Command<'a> { Command::UploadPart { .. } => "text/plain".into(), Command::CreateBucket { .. } => "text/plain".into(), Command::GetObjectAttributes { .. } => "text/plain".into(), + Command::DeleteObjects { .. } => "application/xml".into(), } } @@ -353,6 +359,11 @@ impl<'a> Command<'a> { Command::UploadPart { .. } => EMPTY_PAYLOAD_SHA.into(), Command::InitiateMultipartUpload { .. } => EMPTY_PAYLOAD_SHA.into(), Command::GetObjectAttributes { .. } => EMPTY_PAYLOAD_SHA.into(), + Command::DeleteObjects { data } => { + let mut sha = Sha256::default(); + sha.update(data.to_string().as_bytes()); + hex::encode(sha.finalize().as_slice()) + } }; Ok(result) } diff --git a/s3/src/request/request_trait.rs b/s3/src/request/request_trait.rs index 9199689102..5686cc8dda 100644 --- a/s3/src/request/request_trait.rs +++ b/s3/src/request/request_trait.rs @@ -249,6 +249,8 @@ pub trait Request { } else if let Command::PutBucketCors { configuration, .. } = &self.command() { let cors = configuration.to_string(); cors.as_bytes().to_vec() + } else if let Command::DeleteObjects { data } = &self.command() { + data.to_string().as_bytes().to_vec() } else { Vec::new() }; @@ -550,6 +552,9 @@ pub trait Request { Command::PutObjectTagging { .. } => {} Command::UploadPart { .. } => {} Command::CreateBucket { .. } => {} + Command::DeleteObjects { .. } => { + url_str.push_str("?delete"); + } } let mut url = Url::parse(&url_str)?; @@ -813,6 +818,11 @@ pub trait Request { HeaderName::from_static("x-amz-object-attributes"), "ETag".parse()?, ); + } else if let Command::DeleteObjects { ref data } = self.command() { + let body = data.to_string(); + let digest = md5::compute(body.as_bytes()); + let hash = general_purpose::STANDARD.encode(digest.as_ref()); + headers.insert(HeaderName::from_static("content-md5"), hash.parse()?); } // This must be last, as it signs the other headers, omitted if no secret key is provided diff --git a/s3/src/serde_types.rs b/s3/src/serde_types.rs index a3bf1909df..680ddaebeb 100644 --- a/s3/src/serde_types.rs +++ b/s3/src/serde_types.rs @@ -404,6 +404,107 @@ pub struct AwsError { pub request_id: String, } +/// Represents a single object identifier for the bulk delete request. +#[derive(Debug, Clone)] +pub struct ObjectIdentifier { + /// The key of the object to delete. + pub key: String, + /// The version ID of the object to delete (optional). + pub version_id: Option, +} + +impl ObjectIdentifier { + pub fn new(key: impl Into) -> Self { + Self { + key: key.into(), + version_id: None, + } + } + + pub fn with_version(key: impl Into, version_id: impl Into) -> Self { + Self { + key: key.into(), + version_id: Some(version_id.into()), + } + } +} + +/// Request body for the DeleteObjects (bulk delete) API. +#[derive(Debug, Clone)] +pub struct DeleteObjectsRequest { + pub objects: Vec, + /// If true, the response only includes errors (Quiet mode). + pub quiet: bool, +} + +impl fmt::Display for DeleteObjectsRequest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + r#""# + ) + .expect("Can't fail"); + if self.quiet { + write!(f, "true").expect("Can't fail"); + } + for obj in &self.objects { + let escaped_key = quick_xml::escape::escape(&obj.key); + write!(f, "{}", escaped_key).expect("Can't fail"); + if let Some(ref vid) = obj.version_id { + let escaped_vid = quick_xml::escape::escape(vid); + write!(f, "{}", escaped_vid).expect("Can't fail"); + } + write!(f, "").expect("Can't fail"); + } + write!(f, "") + } +} + +impl DeleteObjectsRequest { + pub fn len(&self) -> usize { + self.to_string().len() + } + + pub fn is_empty(&self) -> bool { + self.objects.is_empty() + } +} + +/// A single deleted object in the DeleteObjects response. +#[derive(Deserialize, Debug, Clone)] +pub struct DeletedObject { + #[serde(rename = "Key")] + pub key: String, + #[serde(rename = "VersionId")] + pub version_id: Option, + #[serde(rename = "DeleteMarker")] + pub delete_marker: Option, + #[serde(rename = "DeleteMarkerVersionId")] + pub delete_marker_version_id: Option, +} + +/// A single error entry in the DeleteObjects response. +#[derive(Deserialize, Debug, Clone)] +pub struct DeleteError { + #[serde(rename = "Key")] + pub key: String, + #[serde(rename = "Code")] + pub code: String, + #[serde(rename = "Message")] + pub message: String, + #[serde(rename = "VersionId")] + pub version_id: Option, +} + +/// The parsed result of a DeleteObjects (bulk delete) response. +#[derive(Deserialize, Debug, Clone)] +pub struct DeleteObjectsResult { + #[serde(rename = "Deleted", default)] + pub deleted: Vec, + #[serde(rename = "Error", default)] + pub errors: Vec, +} + #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] #[serde(rename = "CORSConfiguration")] pub struct CorsConfiguration { @@ -808,7 +909,9 @@ mod test { LifecycleRule, NoncurrentVersionExpiration, NoncurrentVersionTransition, Transition, }; - use super::{CorsConfiguration, CorsRule}; + use super::{ + CorsConfiguration, CorsRule, DeleteObjectsRequest, DeleteObjectsResult, ObjectIdentifier, + }; #[test] fn cors_config_serde() { @@ -876,4 +979,106 @@ mod test { r#"302024-06-01730true1050lala30303030GLACIEREnabled2024-06-01730GLACIER"# ) } + + #[test] + fn delete_objects_request_serialize() { + let request = DeleteObjectsRequest { + objects: vec![ + ObjectIdentifier::new("file1.txt"), + ObjectIdentifier::new("file2.txt"), + ObjectIdentifier::with_version("file3.txt", "v1"), + ], + quiet: false, + }; + + let se = request.to_string(); + assert_eq!( + se, + r#"file1.txtfile2.txtfile3.txtv1"# + ) + } + + #[test] + fn delete_objects_request_serialize_quiet() { + let request = DeleteObjectsRequest { + objects: vec![ObjectIdentifier::new("file1.txt")], + quiet: true, + }; + + let se = request.to_string(); + assert_eq!( + se, + r#"truefile1.txt"# + ) + } + + #[test] + fn delete_objects_request_xml_escaping() { + let request = DeleteObjectsRequest { + objects: vec![ObjectIdentifier::new("file&name<>.txt")], + quiet: false, + }; + + let se = request.to_string(); + assert_eq!( + se, + r#"file&name<>.txt"# + ) + } + + #[test] + fn delete_objects_request_len() { + let request = DeleteObjectsRequest { + objects: vec![ObjectIdentifier::new("file1.txt")], + quiet: false, + }; + + assert_eq!(request.len(), request.to_string().len()); + assert!(!request.is_empty()); + } + + #[test] + fn delete_objects_result_deserialize() { + let xml = r#" + + + file1.txt + + + file2.txt + true + abc123 + + + file3.txt + AccessDenied + Access Denied + +"#; + + let result: DeleteObjectsResult = quick_xml::de::from_str(xml).unwrap(); + assert_eq!(result.deleted.len(), 2); + assert_eq!(result.deleted[0].key, "file1.txt"); + assert_eq!(result.deleted[1].key, "file2.txt"); + assert_eq!(result.deleted[1].delete_marker, Some(true)); + assert_eq!( + result.deleted[1].delete_marker_version_id, + Some("abc123".to_string()) + ); + assert_eq!(result.errors.len(), 1); + assert_eq!(result.errors[0].key, "file3.txt"); + assert_eq!(result.errors[0].code, "AccessDenied"); + assert_eq!(result.errors[0].message, "Access Denied"); + } + + #[test] + fn delete_objects_result_deserialize_empty() { + let xml = r#" + +"#; + + let result: DeleteObjectsResult = quick_xml::de::from_str(xml).unwrap(); + assert_eq!(result.deleted.len(), 0); + assert_eq!(result.errors.len(), 0); + } }