/*
* MinIO .NET Library for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System.Diagnostics.CodeAnalysis;
using Minio.DataModel;
using Minio.DataModel.Args;
using Minio.DataModel.Response;
using Minio.Exceptions;
using Minio.Helper;
namespace Minio;
[SuppressMessage("Design", "MA0048:File name must match type name", Justification = "Split up in partial classes")]
public partial class MinioClient : IMinioClient
{
///
/// private helper method to remove list of objects from bucket
///
/// GetObjectArgs Arguments Object encapsulates information like - bucket name, object name etc
/// Optional cancellation token to cancel the operation
private async Task GetObjectHelper(GetObjectArgs args, CancellationToken cancellationToken = default)
{
// StatObject is called to both verify the existence of the object and return it with GetObject.
// NOTE: This avoids writing the error body to the action stream passed (Do not remove).
var statArgs = new StatObjectArgs()
.WithBucket(args.BucketName)
.WithObject(args.ObjectName)
.WithVersionId(args.VersionId)
.WithMatchETag(args.MatchETag)
.WithNotMatchETag(args.NotMatchETag)
.WithModifiedSince(args.ModifiedSince)
.WithUnModifiedSince(args.UnModifiedSince)
.WithServerSideEncryption(args.SSE)
.WithHeaders(args.Headers);
if (args.OffsetLengthSet) _ = statArgs.WithOffsetAndLength(args.ObjectOffset, args.ObjectLength);
var objStat = await StatObjectAsync(statArgs, cancellationToken).ConfigureAwait(false);
args?.Validate();
if (args.FileName is not null)
await GetObjectFileAsync(args, objStat, cancellationToken).ConfigureAwait(false);
else await GetObjectStreamAsync(args, cancellationToken).ConfigureAwait(false);
return objStat;
}
///
/// private helper method return the specified object from the bucket
///
/// GetObjectArgs Arguments Object encapsulates information like - bucket name, object name etc
/// ObjectStat object encapsulates information like - object name, size, etag etc
/// Optional cancellation token to cancel the operation
private Task GetObjectFileAsync(GetObjectArgs args, ObjectStat objectStat,
CancellationToken cancellationToken = default)
{
var length = objectStat.Size;
var etag = objectStat.ETag;
var tempFileName = $"{args.FileName}.{etag}.part.minio";
if (!string.IsNullOrEmpty(args.VersionId)) tempFileName = $"{args.FileName}.{etag}.{args.VersionId}.part.minio";
if (File.Exists(args.FileName)) File.Delete(args.FileName);
Utils.ValidateFile(tempFileName);
if (File.Exists(tempFileName)) File.Delete(tempFileName);
async Task callbackAsync(Stream stream, CancellationToken cancellationToken)
{
#if NETSTANDARD
using var dest = new FileStream(tempFileName, FileMode.Create, FileAccess.Write);
await stream.CopyToAsync(dest).ConfigureAwait(false);
#else
var dest = new FileStream(tempFileName, FileMode.Create, FileAccess.Write);
await using (dest.ConfigureAwait(false))
{
await stream.CopyToAsync(dest, cancellationToken).ConfigureAwait(false);
}
#endif
}
#pragma warning disable IDISP001 // Dispose created
var cts = new CancellationTokenSource();
#pragma warning restore IDISP001 // Dispose created
cts.CancelAfter(TimeSpan.FromSeconds(15));
args.WithCallbackStream(async (stream, cancellationToken) =>
{
await callbackAsync(stream, cancellationToken).ConfigureAwait(false);
Utils.MoveWithReplace(tempFileName, args.FileName);
});
return GetObjectStreamAsync(args, cancellationToken);
}
///
/// private helper method. It returns the specified portion or full object from the bucket
///
/// GetObjectArgs Arguments Object encapsulates information like - bucket name, object name etc
/// Optional cancellation token to cancel the operation
private async Task GetObjectStreamAsync(GetObjectArgs args, CancellationToken cancellationToken = default)
{
var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false);
using var response =
await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder,
cancellationToken: cancellationToken).ConfigureAwait(false);
}
///
/// private helper method to remove list of objects from bucket
///
///
/// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects,
/// optional list of versions (for each object) to be deleted
///
/// Optional cancellation token to cancel the operation
///
/// When access or secret key provided is invalid
/// When bucket name is invalid
/// When object name is invalid
/// When bucket is not found
/// When object is not found
/// When configuration XML provided is invalid
private async Task> RemoveBucketObjectsAsync(RemoveObjectsArgs args,
CancellationToken cancellationToken)
{
var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false);
using var response =
await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
var removeObjectsResponse = new RemoveObjectsResponse(response.StatusCode, response.Content);
return removeObjectsResponse.DeletedObjectsResult.ErrorList;
}
///
/// private helper method to call remove objects function
///
///
/// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects,
/// optional version Id list
///
/// List of Tuples. Each tuple is Object name to List of Version IDs to be deleted
///
/// Full List of DeleteError objects. The error list from this call will be added to the full
/// list.
///
/// Optional cancellation token to cancel the operation
///
private async Task> CallRemoveObjectVersions(RemoveObjectsArgs args,
IList> objVersions, List fullErrorsList, CancellationToken cancellationToken)
{
var iterArgs = new RemoveObjectsArgs()
.WithBucket(args.BucketName)
.WithObjectsVersions(objVersions);
var errorsList = await RemoveBucketObjectsAsync(iterArgs, cancellationToken).ConfigureAwait(false);
fullErrorsList.AddRange(errorsList);
return fullErrorsList;
}
///
/// private helper method to call function to remove objects/version items in iterations of 1000 each from bucket
///
///
/// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects,
/// optional list of versions (for each object) to be deleted
///
/// List of Object names to be deleted
///
/// Full List of DeleteError objects. The error list from this call will be added to the full
/// list.
///
/// Optional cancellation token to cancel the operation
///
private async Task> CallRemoveObjects(RemoveObjectsArgs args, IList objNames,
List fullErrorsList, CancellationToken cancellationToken)
{
// var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false);
var iterArgs = new RemoveObjectsArgs()
.WithBucket(args.BucketName)
.WithObjects(objNames);
var errorsList = await RemoveBucketObjectsAsync(iterArgs, cancellationToken).ConfigureAwait(false);
fullErrorsList.AddRange(errorsList);
return fullErrorsList;
}
///
/// private helper method to remove objects/version items in iterations of 1000 each from bucket
///
///
/// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects,
/// optional list of versions (for each object) to be deleted
///
///
/// Full List of DeleteError objects. The error list from this call will be added to the full
/// list.
///
/// Optional cancellation token to cancel the operation
///
/// When access or secret key provided is invalid
/// When bucket name is invalid
/// When object name is invalid
/// When bucket is not found
/// When object is not found
/// When configuration XML provided is invalid
private async Task> RemoveObjectVersionsHelper(RemoveObjectsArgs args,
List fullErrorsList, CancellationToken cancellationToken)
{
if (args.ObjectNamesVersions.Count <= 1000)
{
fullErrorsList.AddRange(await CallRemoveObjectVersions(args, args.ObjectNamesVersions, fullErrorsList,
cancellationToken).ConfigureAwait(false));
return fullErrorsList;
}
var curItemList = new List>(args.ObjectNamesVersions.GetRange(0, 1000));
var delVersionNextIndex = curItemList.Count;
var deletedCount = 0;
while (delVersionNextIndex <= args.ObjectNamesVersions.Count)
{
var errorList = await CallRemoveObjectVersions(args, curItemList, fullErrorsList, cancellationToken)
.ConfigureAwait(false);
if (delVersionNextIndex == args.ObjectNamesVersions.Count)
break;
deletedCount += curItemList.Count;
fullErrorsList.AddRange(errorList);
curItemList.Clear();
if (args.ObjectNamesVersions.Count - delVersionNextIndex <= 1000)
{
curItemList.AddRange(args.ObjectNamesVersions.GetRange(delVersionNextIndex,
args.ObjectNamesVersions.Count - delVersionNextIndex));
delVersionNextIndex = args.ObjectNamesVersions.Count;
}
else
{
curItemList.AddRange(args.ObjectNamesVersions.GetRange(delVersionNextIndex, 1000));
delVersionNextIndex += 1000;
}
}
return fullErrorsList;
}
///
/// private helper method to remove objects in iterations of 1000 each from bucket
///
///
/// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects,
/// optional list of versions (for each object) to be deleted
///
///
/// Full List of DeleteError objects. The error list from this call will be added to the full
/// list.
///
/// Optional cancellation token to cancel the operation
///
/// When access or secret key provided is invalid
/// When bucket name is invalid
/// When object name is invalid
/// When bucket is not found
/// When object is not found
/// When configuration XML provided is invalid
private async Task> RemoveObjectsHelper(RemoveObjectsArgs args,
IList fullErrorsList,
CancellationToken cancellationToken)
{
var iterObjects = new List(1000);
var i = 0;
foreach (var objName in args.ObjectNames)
{
Utils.ValidateObjectName(objName);
iterObjects.Insert(i, objName);
if (++i == 1000)
{
fullErrorsList = await CallRemoveObjects(args, iterObjects, fullErrorsList.ToList(), cancellationToken)
.ConfigureAwait(false);
iterObjects.Clear();
i = 0;
}
}
if (iterObjects.Count > 0)
fullErrorsList = await CallRemoveObjects(args, iterObjects, fullErrorsList.ToList(), cancellationToken)
.ConfigureAwait(false);
return fullErrorsList;
}
}