From f5380cffe2a1fbb617ccb5f90a39c0273eb9d50a Mon Sep 17 00:00:00 2001 From: Yan Shkurinsky Date: Tue, 5 Aug 2025 18:01:33 +0400 Subject: [PATCH 1/3] Add IfNoneMatch for PUT req --- .gitignore | 4 +++- src/Network/Minio/Data.hs | 12 ++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 0b4de9a..6735b6c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,6 @@ cabal.sandbox.config *.eventlog .stack-work/ cabal.project.local -*~ \ No newline at end of file +*~ +tags +tags.mtime diff --git a/src/Network/Minio/Data.hs b/src/Network/Minio/Data.hs index 7b60ca0..357d55a 100644 --- a/src/Network/Minio/Data.hs +++ b/src/Network/Minio/Data.hs @@ -373,12 +373,14 @@ data PutObjectOptions = PutObjectOptions -- | Set number of worker threads used to upload an object. pooNumThreads :: Maybe Word, -- | Set object encryption parameters for the request. - pooSSE :: Maybe SSE + pooSSE :: Maybe SSE, + -- | Set a behavior to put object with name already existed in storage. + pooIfNoneMatch :: Maybe Text } -- | Provide default `PutObjectOptions`. defaultPutObjectOptions :: PutObjectOptions -defaultPutObjectOptions = PutObjectOptions Nothing Nothing Nothing Nothing Nothing Nothing [] Nothing Nothing +defaultPutObjectOptions = PutObjectOptions Nothing Nothing Nothing Nothing Nothing Nothing [] Nothing Nothing Nothing pooToHeaders :: PutObjectOptions -> [HT.Header] pooToHeaders poo = @@ -395,7 +397,8 @@ pooToHeaders poo = "content-disposition", "content-language", "cache-control", - "x-amz-storage-class" + "x-amz-storage-class", + "if-none-match" ] values = map @@ -405,7 +408,8 @@ pooToHeaders poo = pooContentDisposition, pooContentLanguage, pooCacheControl, - pooStorageClass + pooStorageClass, + pooIfNoneMatch ] -- | From 1caabd2cda526f82e47439a6e4d9a6f5e7cfdff4 Mon Sep 17 00:00:00 2001 From: Yan Shkurinsky Date: Wed, 6 Aug 2025 16:41:34 +0400 Subject: [PATCH 2/3] import pooIfNoneMatch --- src/Network/Minio.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Network/Minio.hs b/src/Network/Minio.hs index 3cfd9bf..fca8971 100644 --- a/src/Network/Minio.hs +++ b/src/Network/Minio.hs @@ -151,6 +151,7 @@ module Network.Minio pooUserMetadata, pooNumThreads, pooSSE, + pooIfNoneMatch, getObject, GetObjectOptions, defaultGetObjectOptions, From 7bfd5c16007bf74deef29707a7b5f488fe3dcaf4 Mon Sep 17 00:00:00 2001 From: Yan Shkurinsky Date: Wed, 6 Aug 2025 12:25:13 +0400 Subject: [PATCH 3/3] Add putObjectStream --- src/Network/Minio.hs | 16 ++++++++++++++ src/Network/Minio/PutObject.hs | 13 ++++++++++++ src/Network/Minio/S3API.hs | 39 ++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/src/Network/Minio.hs b/src/Network/Minio.hs index fca8971..61f36ec 100644 --- a/src/Network/Minio.hs +++ b/src/Network/Minio.hs @@ -140,6 +140,7 @@ module Network.Minio -- ** Conduit-based streaming operations putObject, + putObjectStream, PutObjectOptions, defaultPutObjectOptions, pooContentType, @@ -285,6 +286,21 @@ putObject :: putObject bucket object src sizeMay opts = void $ putObjectInternal bucket object opts $ ODStream src sizeMay +-- | Put an object from a conduit source. The size need be provided. +-- This version of putting object is optimized to stream data without +-- need to be full resident in memory and then this is suitable +-- to put small objects cause it is never does multipart upload +putObjectStream :: + Bucket -> + Object -> + C.ConduitM () ByteString Minio () -> + Int64 -> + MinioConn -> + PutObjectOptions -> + Minio () +putObjectStream bucket object src size minioConn opts = + void $ putObjectInternalStream bucket object opts src size minioConn + -- | Perform a server-side copy operation to create an object based on -- the destination specification in DestinationInfo from the source -- specification in SourceInfo. This function performs a multipart diff --git a/src/Network/Minio/PutObject.hs b/src/Network/Minio/PutObject.hs index e1a8ff3..227ebd8 100644 --- a/src/Network/Minio/PutObject.hs +++ b/src/Network/Minio/PutObject.hs @@ -16,6 +16,7 @@ module Network.Minio.PutObject ( putObjectInternal, + putObjectInternalStream, ObjectData (..), selectPartSizes, ) @@ -103,6 +104,18 @@ putObjectInternal b o opts (ODFile fp sizeMay) = do sequentialMultipartUpload b o opts (Just size) $ CB.sourceFile fp +-- | Put an object from ObjectData, optimized to put streamly +putObjectInternalStream :: + Bucket -> + Object -> + PutObjectOptions -> + C.ConduitM () ByteString Minio () -> + Int64 -> + MinioConn -> + Minio ETag +putObjectInternalStream b o opts src size minioConn = do + putObjectSingleStream b o (pooToHeaders opts) src size minioConn + parallelMultipartUpload :: Bucket -> Object -> diff --git a/src/Network/Minio/S3API.hs b/src/Network/Minio/S3API.hs index f628b48..4cf3278 100644 --- a/src/Network/Minio/S3API.hs +++ b/src/Network/Minio/S3API.hs @@ -56,6 +56,7 @@ module Network.Minio.S3API maxSinglePutObjectSizeBytes, putObjectSingle', putObjectSingle, + putObjectSingleStream, copyObjectSingle, -- * Multipart Upload APIs @@ -108,6 +109,7 @@ module Network.Minio.S3API where import qualified Data.ByteString as BS +import qualified Conduit as C import qualified Data.Text as T import Lib.Prelude import qualified Network.HTTP.Conduit as NC @@ -229,6 +231,43 @@ putObjectSingle' bucket object headers bs = do return etag + +-- | PUT an object into the service. This function performs a single +-- PUT object call and uses a conduit of ByteString as the object +-- data. +putObjectSingleStream :: + Bucket -> + Object -> + [HT.Header] -> + C.ConduitM () ByteString Minio () -> + Int64 -> + MinioConn -> + Minio ETag +putObjectSingleStream bucket object headers src size minioConn = do + when (size > maxSinglePutObjectSizeBytes) $ + throwIO $ + MErrVSinglePUTSizeExceeded size + + let payload = PayloadC size $ C.transPipe (flip runReaderT minioConn . coerce) src + resp <- + mkStreamRequest $ + defaultS3ReqInfo + { riMethod = HT.methodPut, + riBucket = Just bucket, + riObject = Just object, + riHeaders = headers, + riPayload = payload + } + + let rheaders = NC.responseHeaders resp + etag = getETagHeader rheaders + maybe + (throwIO MErrVETagHeaderNotFound) + return + etag + + + -- | PUT an object into the service. This function performs a single -- PUT object call, and so can only transfer objects upto 5GiB. putObjectSingle ::