/* * MinIO .NET Library for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ using System.Diagnostics.CodeAnalysis; using Minio.DataModel; using Minio.DataModel.Args; using Minio.DataModel.Response; using Minio.Exceptions; using Minio.Helper; namespace Minio; [SuppressMessage("Design", "MA0048:File name must match type name", Justification = "Split up in partial classes")] public partial class MinioClient : IMinioClient { /// /// private helper method to remove list of objects from bucket /// /// GetObjectArgs Arguments Object encapsulates information like - bucket name, object name etc /// Optional cancellation token to cancel the operation private async Task GetObjectHelper(GetObjectArgs args, CancellationToken cancellationToken = default) { // StatObject is called to both verify the existence of the object and return it with GetObject. // NOTE: This avoids writing the error body to the action stream passed (Do not remove). var statArgs = new StatObjectArgs() .WithBucket(args.BucketName) .WithObject(args.ObjectName) .WithVersionId(args.VersionId) .WithMatchETag(args.MatchETag) .WithNotMatchETag(args.NotMatchETag) .WithModifiedSince(args.ModifiedSince) .WithUnModifiedSince(args.UnModifiedSince) .WithServerSideEncryption(args.SSE) .WithHeaders(args.Headers); if (args.OffsetLengthSet) _ = statArgs.WithOffsetAndLength(args.ObjectOffset, args.ObjectLength); var objStat = await StatObjectAsync(statArgs, cancellationToken).ConfigureAwait(false); args?.Validate(); if (args.FileName is not null) await GetObjectFileAsync(args, objStat, cancellationToken).ConfigureAwait(false); else await GetObjectStreamAsync(args, cancellationToken).ConfigureAwait(false); return objStat; } /// /// private helper method return the specified object from the bucket /// /// GetObjectArgs Arguments Object encapsulates information like - bucket name, object name etc /// ObjectStat object encapsulates information like - object name, size, etag etc /// Optional cancellation token to cancel the operation private Task GetObjectFileAsync(GetObjectArgs args, ObjectStat objectStat, CancellationToken cancellationToken = default) { var length = objectStat.Size; var etag = objectStat.ETag; var tempFileName = $"{args.FileName}.{etag}.part.minio"; if (!string.IsNullOrEmpty(args.VersionId)) tempFileName = $"{args.FileName}.{etag}.{args.VersionId}.part.minio"; if (File.Exists(args.FileName)) File.Delete(args.FileName); Utils.ValidateFile(tempFileName); if (File.Exists(tempFileName)) File.Delete(tempFileName); async Task callbackAsync(Stream stream, CancellationToken cancellationToken) { #if NETSTANDARD using var dest = new FileStream(tempFileName, FileMode.Create, FileAccess.Write); await stream.CopyToAsync(dest).ConfigureAwait(false); #else var dest = new FileStream(tempFileName, FileMode.Create, FileAccess.Write); await using (dest.ConfigureAwait(false)) { await stream.CopyToAsync(dest, cancellationToken).ConfigureAwait(false); } #endif } #pragma warning disable IDISP001 // Dispose created var cts = new CancellationTokenSource(); #pragma warning restore IDISP001 // Dispose created cts.CancelAfter(TimeSpan.FromSeconds(15)); args.WithCallbackStream(async (stream, cancellationToken) => { await callbackAsync(stream, cancellationToken).ConfigureAwait(false); Utils.MoveWithReplace(tempFileName, args.FileName); }); return GetObjectStreamAsync(args, cancellationToken); } /// /// private helper method. It returns the specified portion or full object from the bucket /// /// GetObjectArgs Arguments Object encapsulates information like - bucket name, object name etc /// Optional cancellation token to cancel the operation private async Task GetObjectStreamAsync(GetObjectArgs args, CancellationToken cancellationToken = default) { var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false); using var response = await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, cancellationToken: cancellationToken).ConfigureAwait(false); } /// /// private helper method to remove list of objects from bucket /// /// /// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects, /// optional list of versions (for each object) to be deleted /// /// Optional cancellation token to cancel the operation /// /// When access or secret key provided is invalid /// When bucket name is invalid /// When object name is invalid /// When bucket is not found /// When object is not found /// When configuration XML provided is invalid private async Task> RemoveBucketObjectsAsync(RemoveObjectsArgs args, CancellationToken cancellationToken) { var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false); using var response = await this.ExecuteTaskAsync(ResponseErrorHandlers, requestMessageBuilder, cancellationToken: cancellationToken) .ConfigureAwait(false); var removeObjectsResponse = new RemoveObjectsResponse(response.StatusCode, response.Content); return removeObjectsResponse.DeletedObjectsResult.ErrorList; } /// /// private helper method to call remove objects function /// /// /// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects, /// optional version Id list /// /// List of Tuples. Each tuple is Object name to List of Version IDs to be deleted /// /// Full List of DeleteError objects. The error list from this call will be added to the full /// list. /// /// Optional cancellation token to cancel the operation /// private async Task> CallRemoveObjectVersions(RemoveObjectsArgs args, IList> objVersions, List fullErrorsList, CancellationToken cancellationToken) { var iterArgs = new RemoveObjectsArgs() .WithBucket(args.BucketName) .WithObjectsVersions(objVersions); var errorsList = await RemoveBucketObjectsAsync(iterArgs, cancellationToken).ConfigureAwait(false); fullErrorsList.AddRange(errorsList); return fullErrorsList; } /// /// private helper method to call function to remove objects/version items in iterations of 1000 each from bucket /// /// /// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects, /// optional list of versions (for each object) to be deleted /// /// List of Object names to be deleted /// /// Full List of DeleteError objects. The error list from this call will be added to the full /// list. /// /// Optional cancellation token to cancel the operation /// private async Task> CallRemoveObjects(RemoveObjectsArgs args, IList objNames, List fullErrorsList, CancellationToken cancellationToken) { // var requestMessageBuilder = await this.CreateRequest(args).ConfigureAwait(false); var iterArgs = new RemoveObjectsArgs() .WithBucket(args.BucketName) .WithObjects(objNames); var errorsList = await RemoveBucketObjectsAsync(iterArgs, cancellationToken).ConfigureAwait(false); fullErrorsList.AddRange(errorsList); return fullErrorsList; } /// /// private helper method to remove objects/version items in iterations of 1000 each from bucket /// /// /// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects, /// optional list of versions (for each object) to be deleted /// /// /// Full List of DeleteError objects. The error list from this call will be added to the full /// list. /// /// Optional cancellation token to cancel the operation /// /// When access or secret key provided is invalid /// When bucket name is invalid /// When object name is invalid /// When bucket is not found /// When object is not found /// When configuration XML provided is invalid private async Task> RemoveObjectVersionsHelper(RemoveObjectsArgs args, List fullErrorsList, CancellationToken cancellationToken) { if (args.ObjectNamesVersions.Count <= 1000) { fullErrorsList.AddRange(await CallRemoveObjectVersions(args, args.ObjectNamesVersions, fullErrorsList, cancellationToken).ConfigureAwait(false)); return fullErrorsList; } var curItemList = new List>(args.ObjectNamesVersions.GetRange(0, 1000)); var delVersionNextIndex = curItemList.Count; var deletedCount = 0; while (delVersionNextIndex <= args.ObjectNamesVersions.Count) { var errorList = await CallRemoveObjectVersions(args, curItemList, fullErrorsList, cancellationToken) .ConfigureAwait(false); if (delVersionNextIndex == args.ObjectNamesVersions.Count) break; deletedCount += curItemList.Count; fullErrorsList.AddRange(errorList); curItemList.Clear(); if (args.ObjectNamesVersions.Count - delVersionNextIndex <= 1000) { curItemList.AddRange(args.ObjectNamesVersions.GetRange(delVersionNextIndex, args.ObjectNamesVersions.Count - delVersionNextIndex)); delVersionNextIndex = args.ObjectNamesVersions.Count; } else { curItemList.AddRange(args.ObjectNamesVersions.GetRange(delVersionNextIndex, 1000)); delVersionNextIndex += 1000; } } return fullErrorsList; } /// /// private helper method to remove objects in iterations of 1000 each from bucket /// /// /// RemoveObjectsArgs Arguments Object encapsulates information like - bucket name, List of objects, /// optional list of versions (for each object) to be deleted /// /// /// Full List of DeleteError objects. The error list from this call will be added to the full /// list. /// /// Optional cancellation token to cancel the operation /// /// When access or secret key provided is invalid /// When bucket name is invalid /// When object name is invalid /// When bucket is not found /// When object is not found /// When configuration XML provided is invalid private async Task> RemoveObjectsHelper(RemoveObjectsArgs args, IList fullErrorsList, CancellationToken cancellationToken) { var iterObjects = new List(1000); var i = 0; foreach (var objName in args.ObjectNames) { Utils.ValidateObjectName(objName); iterObjects.Insert(i, objName); if (++i == 1000) { fullErrorsList = await CallRemoveObjects(args, iterObjects, fullErrorsList.ToList(), cancellationToken) .ConfigureAwait(false); iterObjects.Clear(); i = 0; } } if (iterObjects.Count > 0) fullErrorsList = await CallRemoveObjects(args, iterObjects, fullErrorsList.ToList(), cancellationToken) .ConfigureAwait(false); return fullErrorsList; } }