Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion include/miniocpp/args.h
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ struct ComposeSource : public ObjectConditionalReadArgs {
utils::Multimap Headers() const;

private:
long object_size_ = -1;
std::optional<size_t> object_size_;
utils::Multimap headers_;
}; // struct ComposeSource

Expand Down
12 changes: 7 additions & 5 deletions include/miniocpp/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <map>
#include <memory>
#include <nlohmann/json_fwd.hpp>
#include <optional>
#include <ostream>
#include <string>
#include <type_traits>
Expand Down Expand Up @@ -293,17 +294,18 @@ struct SelectRequest {
struct SelectResult {
error::Error err = error::SUCCESS;
bool ended = false;
long int bytes_scanned = -1;
long int bytes_processed = -1;
long int bytes_returned = -1;
std::optional<long long> bytes_scanned;
std::optional<long long> bytes_processed;
std::optional<long long> bytes_returned;
std::string records;

SelectResult() : ended(true) {}

explicit SelectResult(error::Error err) : err(std::move(err)), ended(true) {}

SelectResult(long int bytes_scanned, long int bytes_processed,
long int bytes_returned)
SelectResult(std::optional<long long> bytes_scanned,
std::optional<long long> bytes_processed,
std::optional<long long> bytes_returned)
: bytes_scanned(bytes_scanned),
bytes_processed(bytes_processed),
bytes_returned(bytes_returned) {}
Expand Down
6 changes: 3 additions & 3 deletions src/args.cc
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ error::Error ComposeSource::BuildHeaders(size_t object_size,
}
}

object_size_ = static_cast<long>(object_size);
object_size_ = object_size;
headers_ = CopyHeaders();
if (!headers_.Contains("x-amz-copy-source-if-match")) {
headers_.Add("x-amz-copy-source-if-match", etag);
Expand All @@ -391,14 +391,14 @@ error::Error ComposeSource::BuildHeaders(size_t object_size,
}

size_t ComposeSource::ObjectSize() const {
if (object_size_ == -1) {
if (!object_size_.has_value()) {
std::cerr << "ABORT: ComposeSource::BuildHeaders() must be called prior to "
"this method invocation. This should not happen."
<< std::endl;
std::terminate();
}

return object_size_;
return *object_size_;
}

utils::Multimap ComposeSource::Headers() const {
Expand Down
2 changes: 1 addition & 1 deletion src/baseclient.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2049,7 +2049,7 @@ Result<StatObjectResponse> BaseClient::StatObject(StatObjectArgs args) {
resp.etag = utils::Trim(response->headers.GetFront("etag"), '"');

std::string value = response->headers.GetFront("content-length");
if (!value.empty()) resp.size = std::stol(value);
if (!value.empty()) resp.size = std::stoll(value);

value = response->headers.GetFront("last-modified");
if (!value.empty()) {
Expand Down
3 changes: 2 additions & 1 deletion src/client.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1165,7 +1165,8 @@ Result<DownloadObjectResponse> Client::DownloadObject(DownloadObjectArgs args) {

std::string temp_filename =
args.filename + "." + curlpp::escape(etag) + ".part.minio";
std::ofstream fout(temp_filename, std::ios::trunc | std::ios::out);
std::ofstream fout(temp_filename,
std::ios::trunc | std::ios::out | std::ios::binary);
if (!fout.is_open()) {
return error::make<DownloadObjectResponse>("unable to open file " +
temp_filename);
Expand Down
12 changes: 6 additions & 6 deletions src/select.cc
Original file line number Diff line number Diff line change
Expand Up @@ -195,21 +195,21 @@ bool SelectHandler::process(const http::DataFunctionArgs& /* args */,
auto root = xdoc.select_node(xpath.c_str());
pugi::xpath_node text;
std::string value;
long int bytes_scanned = -1;
long int bytes_processed = -1;
long int bytes_returned = -1;
std::optional<long long> bytes_scanned;
std::optional<long long> bytes_processed;
std::optional<long long> bytes_returned;

text = root.node().select_node("BytesScanned/text()");
value = text.node().value();
if (!value.empty()) bytes_scanned = std::stol(value);
if (!value.empty()) bytes_scanned = std::stoll(value);

text = root.node().select_node("BytesProcessed/text()");
value = text.node().value();
if (!value.empty()) bytes_processed = std::stol(value);
if (!value.empty()) bytes_processed = std::stoll(value);

text = root.node().select_node("BytesReturned/text()");
value = text.node().value();
if (!value.empty()) bytes_returned = std::stol(value);
if (!value.empty()) bytes_returned = std::stoll(value);

cont = result_func_(
SelectResult(bytes_scanned, bytes_processed, bytes_returned));
Expand Down
92 changes: 92 additions & 0 deletions tests/tests.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <miniocpp/request.h>
#include <miniocpp/response.h>
#include <miniocpp/result.h>
#include <miniocpp/select.h>
#include <miniocpp/types.h>

using minio::Result;
Expand All @@ -38,6 +39,7 @@ using minio::Result;
#include <iosfwd>
#include <iostream>
#include <list>
#include <map>
#include <ostream>
#include <random>
#include <sstream>
Expand Down Expand Up @@ -102,6 +104,43 @@ std::string RandBucketName() {

std::string RandObjectName() { return RandomString(charset, 8); }

std::string PutUint32BigEndian(unsigned int v) {
std::string s(4, '\0');
s[0] = static_cast<char>((v >> 24) & 0xFF);
s[1] = static_cast<char>((v >> 16) & 0xFF);
s[2] = static_cast<char>((v >> 8) & 0xFF);
s[3] = static_cast<char>(v & 0xFF);
return s;
}

// Build a single S3 Select protocol frame (prelude + prelude CRC + headers
// + payload + message CRC) for the given event headers and XML payload.
std::string MakeSelectFrame(const std::map<std::string, std::string>& headers,
const std::string& payload) {
std::string headerdata;
for (const auto& [name, value] : headers) {
headerdata += static_cast<char>(name.length());
headerdata += name;
headerdata += static_cast<char>(7); // header value type: string
headerdata += static_cast<char>((value.length() >> 8) & 0xFF);
headerdata += static_cast<char>(value.length() & 0xFF);
headerdata += value;
}
headerdata += static_cast<char>(0); // header terminator

std::string data = headerdata + payload;
unsigned int total_length = 16 + static_cast<unsigned int>(data.length());
std::string prelude =
PutUint32BigEndian(total_length) +
PutUint32BigEndian(static_cast<unsigned int>(headerdata.length()));
std::string prelude_crc = PutUint32BigEndian(
static_cast<unsigned int>(minio::utils::CRC32(prelude)));
std::string message = prelude + prelude_crc + data;
std::string message_crc = PutUint32BigEndian(
static_cast<unsigned int>(minio::utils::CRC32(message)));
return message + message_crc;
}

struct MakeBucketError : public std::runtime_error {
MakeBucketError(std::string err) : runtime_error(err) {}
};
Expand Down Expand Up @@ -1537,6 +1576,58 @@ class Tests {
RemoveObject(bucket_name_, object_name);
}
} // TestAsyncOperations

// Regression test for SelectHandler Stats metric parsing: metrics larger
// than INT32_MAX must round-trip as exact long long values (any fallback
// to std::stol on 32-bit Windows LLP64 would truncate them). Uses a
// synthetic Stats event frame, independent of large objects.
void SelectStatsMetrics() {
std::cout << "SelectStatsMetrics()" << std::endl;

const long long scanned = 5000000000LL;
const long long processed = 6000000000LL;
const long long returned = 7000000000LL;

std::map<std::string, std::string> headers = {
{":message-type", "event"},
{":event-type", "Stats"},
};
std::string payload = "<Stats><BytesScanned>" + std::to_string(scanned) +
"</BytesScanned><BytesProcessed>" +
std::to_string(processed) +
"</BytesProcessed><BytesReturned>" +
std::to_string(returned) + "</BytesReturned></Stats>";

bool stats_delivered = false;
minio::s3::SelectHandler handler(
[&](minio::s3::SelectResult result) -> bool {
if (result.err) {
throw std::runtime_error("SelectStatsMetrics(): " +
result.err.String());
}
if (!result.bytes_scanned.has_value() ||
*result.bytes_scanned != scanned ||
!result.bytes_processed.has_value() ||
*result.bytes_processed != processed ||
!result.bytes_returned.has_value() ||
*result.bytes_returned != returned) {
throw std::runtime_error(
"SelectStatsMetrics(): unexpected metrics");
}
stats_delivered = true;
return true;
});

minio::http::DataFunctionArgs args;
args.datachunk = MakeSelectFrame(headers, payload);
if (!handler.DataFunction(args)) {
throw std::runtime_error("SelectStatsMetrics(): DataFunction failed");
}
if (!stats_delivered) {
throw std::runtime_error(
"SelectStatsMetrics(): Stats result was not delivered");
}
}
}; // class Tests

int main(int /*argc*/, char* /*argv*/[]) {
Expand Down Expand Up @@ -1596,6 +1687,7 @@ int main(int /*argc*/, char* /*argv*/[]) {
tests.SelectObjectContent();
tests.ListenBucketNotification();
tests.TestAsyncOperations();
tests.SelectStatsMetrics();

return EXIT_SUCCESS;
}
Loading