Skip to content

Fix/gcp etag issue #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vsazhenyuk-softheme merged 3 commits into from
Jul 31, 2026
Merged

Fix/gcp etag issue #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
542f306
Arbitrary S3 object versions support
vsazhenyuk-softheme Feb 21, 2026
7baa0e2
fix rebase
zoryamba-elastio Jul 29, 2026
95c5038
update submodule
zoryamba-elastio Jul 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@
url = https://github.com/awslabs/aws-c-auth.git
[submodule "aws-c-s3"]
path = mountpoint-s3-crt-sys/crt/aws-c-s3
url = https://github.com/awslabs/aws-c-s3.git
url = https://github.com/elastio/aws-c-s3.git
22 changes: 22 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ members = [
"mountpoint-s3",
"mountpoint-s3-fuser",
"mountpoint-s3-fs",
"mountpoint-s3-ioctl",
]
resolver = "2"

Expand Down
2 changes: 1 addition & 1 deletion mountpoint-s3-client/examples/client_benchmark.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ fn run_benchmark(
futures::executor::block_on(async move {
let mut received_obj_len = 0u64;
let mut request = client
.get_object(bucket, key, &GetObjectParams::new())
.get_object(bucket, key, None, &GetObjectParams::new())
.await
.expect("couldn't create get request");
let mut backpressure_handle = request.backpressure_handle().cloned();
Expand Down
2 changes: 1 addition & 1 deletion mountpoint-s3-client/examples/download.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ fn main() {
let last_offset_clone = Arc::clone(&last_offset);
futures::executor::block_on(async move {
let mut request = client
.get_object(bucket, key, &GetObjectParams::new().range(range))
.get_object(bucket, key, None, &GetObjectParams::new().range(range))
.await
.expect("couldn't create get request");
loop {
Expand Down
8 changes: 5 additions & 3 deletions mountpoint-s3-client/src/failure_client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,15 @@ where
&self,
bucket: &str,
key: &str,
version: Option<&str>,
params: &GetObjectParams,
) -> ObjectClientResult<Self::GetObjectResponse, GetObjectError, Self::ClientError> {
let failure_mode = (self.get_object_cb)(&mut *self.state.lock().unwrap(), bucket, key, params);
if let Some(GetObjectFailureMode::OperationError(err)) = failure_mode {
return Err(err);
}

let request = self.client.get_object(bucket, key, params).await?;
let request = self.client.get_object(bucket, key, version, params).await?;
Ok(FailureGetResponse {
request,
poll_count: 0,
Expand Down Expand Up @@ -156,10 +157,11 @@ where
&self,
bucket: &str,
key: &str,
version: Option<&str>,
params: &HeadObjectParams,
) -> ObjectClientResult<HeadObjectResult, HeadObjectError, Self::ClientError> {
(self.head_object_cb)(&mut *self.state.lock().unwrap(), bucket, key)?;
self.client.head_object(bucket, key, params).await
self.client.head_object(bucket, key, version, params).await
}

async fn put_object(
Expand Down Expand Up @@ -491,7 +493,7 @@ mod tests {

let fail_set = HashSet::from([2, 4, 5]);
for i in 1..=6 {
let r = fail_client.get_object(bucket, key, &GetObjectParams::new()).await;
let r = fail_client.get_object(bucket, key, None, &GetObjectParams::new()).await;
if fail_set.contains(&i) {
assert!(r.is_err());
} else {
Expand Down
2 changes: 1 addition & 1 deletion mountpoint-s3-client/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
//!
//! let client = S3CrtClient::new(Default::default()).expect("client construction failed");
//!
//! let response = client.get_object("my-bucket", "my-key", &GetObjectParams::new()).await.expect("get_object failed");
//! let response = client.get_object("my-bucket", "my-key", Some("object-version"), &GetObjectParams::new()).await.expect("get_object failed");
//! let body = response.map_ok(|part| part.data.to_vec()).try_concat().await.expect("body streaming failed");
//! # }
//! ```
Expand Down
80 changes: 59 additions & 21 deletions mountpoint-s3-client/src/mock_client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -936,6 +936,7 @@ impl ObjectClient for MockClient {
&self,
bucket: &str,
key: &str,
_version: Option<&str>,
params: &GetObjectParams,
) -> ObjectClientResult<Self::GetObjectResponse, GetObjectError, Self::ClientError> {
trace!(bucket, key, ?params.range, ?params.if_match, "GetObject");
Expand Down Expand Up @@ -999,6 +1000,7 @@ impl ObjectClient for MockClient {
&self,
bucket: &str,
key: &str,
_version: Option<&str>,
params: &HeadObjectParams,
) -> ObjectClientResult<HeadObjectResult, HeadObjectError, Self::ClientError> {
trace!(bucket, key, "HeadObject");
Expand All @@ -1025,6 +1027,7 @@ impl ObjectClient for MockClient {
checksum,
sse_type: None,
sse_kms_key_id: None,
version_id: None,
})
} else {
Err(ObjectClientError::ServiceError(HeadObjectError::NotFound))
Expand Down Expand Up @@ -1455,7 +1458,7 @@ mod tests {
client.add_object(key, object);

let mut get_request = client
.get_object("test_bucket", key, &GetObjectParams::new().range(range.clone()))
.get_object("test_bucket", key, None, &GetObjectParams::new().range(range.clone()))
.await
.expect("should not fail");

Expand Down Expand Up @@ -1509,7 +1512,7 @@ mod tests {
client.add_object(key, MockObject::from_bytes(&body, ETag::for_tests()));

let mut get_request = client
.get_object("test_bucket", key, &GetObjectParams::new().range(range.clone()))
.get_object("test_bucket", key, None, &GetObjectParams::new().range(range.clone()))
.await
.expect("should not fail");
let mut backpressure_handle = get_request
Expand Down Expand Up @@ -1553,44 +1556,71 @@ mod tests {
client.add_object("key1", body[..].into());

assert!(matches!(
client.get_object("wrong_bucket", "key1", &GetObjectParams::new()).await,
client
.get_object("wrong_bucket", "key1", None, &GetObjectParams::new())
.await,
Err(ObjectClientError::ServiceError(GetObjectError::NoSuchBucket(_)))
));

assert!(matches!(
client
.get_object("test_bucket", "wrong_key", &GetObjectParams::new())
.get_object("test_bucket", "wrong_key", None, &GetObjectParams::new())
.await,
Err(ObjectClientError::ServiceError(GetObjectError::NoSuchKey(_)))
));

assert_client_error!(
client
.get_object("test_bucket", "key1", &GetObjectParams::new().range(Some(0..2001)))
.get_object(
"test_bucket",
"key1",
None,
&GetObjectParams::new().range(Some(0..2001))
)
.await,
"invalid range, length=2000"
);
assert_client_error!(
client
.get_object("test_bucket", "key1", &GetObjectParams::new().range(Some(2000..2000)))
.get_object(
"test_bucket",
"key1",
None,
&GetObjectParams::new().range(Some(2000..2000))
)
.await,
"invalid range, length=2000"
);
assert_client_error!(
client
.get_object("test_bucket", "key1", &GetObjectParams::new().range(Some(500..2001)))
.get_object(
"test_bucket",
"key1",
None,
&GetObjectParams::new().range(Some(500..2001))
)
.await,
"invalid range, length=2000"
);
assert_client_error!(
client
.get_object("test_bucket", "key1", &GetObjectParams::new().range(Some(5000..2001)))
.get_object(
"test_bucket",
"key1",
None,
&GetObjectParams::new().range(Some(5000..2001))
)
.await,
"invalid range, length=2000"
);
assert_client_error!(
client
.get_object("test_bucket", "key1", &GetObjectParams::new().range(Some(5000..1)))
.get_object(
"test_bucket",
"key1",
None,
&GetObjectParams::new().range(Some(5000..1))
)
.await,
"invalid range, length=2000"
);
Expand Down Expand Up @@ -1619,7 +1649,12 @@ mod tests {
client.add_object(key, MockObject::from_bytes(&expected_body, ETag::for_tests()));

let mut get_request = client
.get_object("test_bucket", key, &GetObjectParams::new().range(Some(range.clone())))
.get_object(
"test_bucket",
key,
None,
&GetObjectParams::new().range(Some(range.clone())),
)
.await
.expect("should not fail");

Expand Down Expand Up @@ -1652,7 +1687,7 @@ mod tests {
.expect("Should not fail");

client
.get_object(bucket, dst_key, &GetObjectParams::new())
.get_object(bucket, dst_key, None, &GetObjectParams::new())
.await
.expect("get_object should succeed");
}
Expand Down Expand Up @@ -2059,7 +2094,7 @@ mod tests {
put_request.complete().await.expect("put_object failed");

let mut get_request = client
.get_object("test_bucket", "key1", &GetObjectParams::new())
.get_object("test_bucket", "key1", None, &GetObjectParams::new())
.await
.expect("get_object failed");

Expand Down Expand Up @@ -2092,7 +2127,7 @@ mod tests {
.expect("put_object failed");

let get_request = client
.get_object("test_bucket", "key1", &GetObjectParams::new())
.get_object("test_bucket", "key1", None, &GetObjectParams::new())
.await
.expect("get_object failed");

Expand Down Expand Up @@ -2284,12 +2319,12 @@ mod tests {
.expect("rename should succeed");
assert!(matches!(
client
.head_object("test_bucket", "key1", &HeadObjectParams::new())
.head_object("test_bucket", "key1", None, &HeadObjectParams::new())
.await,
Err(ObjectClientError::ServiceError(HeadObjectError::NotFound))
));
client
.head_object("test_bucket", "new_key1", &HeadObjectParams::new())
.head_object("test_bucket", "new_key1", None, &HeadObjectParams::new())
.await
.expect("object should now exist with new key");
}
Expand Down Expand Up @@ -2347,12 +2382,12 @@ mod tests {
// Assert that key2 is accessible, while key1 is not
assert!(matches!(
client
.head_object("test_bucket", "key1", &HeadObjectParams::new())
.head_object("test_bucket", "key1", None, &HeadObjectParams::new())
.await,
Err(ObjectClientError::ServiceError(HeadObjectError::NotFound))
));
client
.head_object("test_bucket", "key2", &HeadObjectParams::new())
.head_object("test_bucket", "key2", None, &HeadObjectParams::new())
.await
.expect("object should now exist with new key");
}
Expand Down Expand Up @@ -2391,7 +2426,10 @@ mod tests {
put_request.complete().await.unwrap();

// head_object returns storage class
let head_result = client.head_object(bucket, key, &HeadObjectParams::new()).await.unwrap();
let head_result = client
.head_object(bucket, key, None, &HeadObjectParams::new())
.await
.unwrap();
assert_eq!(head_result.storage_class.as_deref(), storage_class);

// list_objects returns storage class
Expand All @@ -2409,14 +2447,14 @@ mod tests {
let head_counter_1 = client.new_counter(Operation::HeadObject);
let delete_counter_1 = client.new_counter(Operation::DeleteObject);

let _result = client.head_object(bucket, "key", &HeadObjectParams::new()).await;
let _result = client.head_object(bucket, "key", None, &HeadObjectParams::new()).await;
assert_eq!(1, head_counter_1.count());
assert_eq!(0, delete_counter_1.count());

let head_counter_2 = client.new_counter(Operation::HeadObject);
assert_eq!(0, head_counter_2.count());

let _result = client.head_object(bucket, "key", &HeadObjectParams::new()).await;
let _result = client.head_object(bucket, "key", None, &HeadObjectParams::new()).await;
let _result = client.delete_object(bucket, "key").await;
let _result = client.delete_object(bucket, "key").await;
let _result = client.delete_object(bucket, "key").await;
Expand Down Expand Up @@ -2547,7 +2585,7 @@ mod tests {
.expect("append failed");

let get_request = client
.get_object(bucket, key, &GetObjectParams::default())
.get_object(bucket, key, None, &GetObjectParams::default())
.await
.expect("get_object failed");

Expand Down
8 changes: 5 additions & 3 deletions mountpoint-s3-client/src/mock_client/throughput_client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,10 @@ impl ObjectClient for ThroughputMockClient {
&self,
bucket: &str,
key: &str,
version: Option<&str>,
params: &GetObjectParams,
) -> ObjectClientResult<Self::GetObjectResponse, GetObjectError, Self::ClientError> {
let request = self.inner.get_object(bucket, key, params).await?;
let request = self.inner.get_object(bucket, key, version, params).await?;
let rate_limiter = self.rate_limiter.clone();
Ok(ThroughputGetObjectResponse { request, rate_limiter })
}
Expand All @@ -188,9 +189,10 @@ impl ObjectClient for ThroughputMockClient {
&self,
bucket: &str,
key: &str,
version: Option<&str>,
params: &HeadObjectParams,
) -> ObjectClientResult<HeadObjectResult, HeadObjectError, Self::ClientError> {
self.inner.head_object(bucket, key, params).await
self.inner.head_object(bucket, key, version, params).await
}

async fn put_object(
Expand Down Expand Up @@ -267,7 +269,7 @@ mod tests {
let num_bytes = block_on(async move {
let mut num_bytes = 0;
let mut get = client
.get_object("test_bucket", "testfile", &GetObjectParams::new())
.get_object("test_bucket", "testfile", None, &GetObjectParams::new())
.await
.unwrap();
while let Some(part) = get.next().await {
Expand Down
Loading
Loading