I am using a file upload component that posts the file to an API controller and this works ok but i need to get the progres of the upload.
[HttpPost("upload/single")]
public async Task SingleAsync(IFormFile file)
{
try
{
// Azure connection string and container name passed as an argument to get the Blob reference of the container.
var container = new BlobContainerClient(azureConnectionString, "upload-container");
// Method to create our container if it doesn’t exist.
var createResponse = await container.CreateIfNotExistsAsync();
// If container successfully created, then set public access type to Blob.
if (createResponse != null && createResponse.GetRawResponse().Status == 201)
await container.SetAccessPolicyAsync(Azure.Storage.Blobs.Models.PublicAccessType.Blob);
// Method to create a new Blob client.
var blob = container.GetBlobClient(file.FileName);
// If a blob with the same name exists, then we delete the Blob and its snapshots.
await blob.DeleteIfExistsAsync(Azure.Storage.Blobs.Models.DeleteSnapshotsOption.IncludeSnapshots);
// Create a file stream and use the UploadSync method to upload the Blob.
uploadFileSize = file.Length;
var progressHandler = new Progress<long>();
progressHandler.ProgressChanged += UploadProgressChanged;
using (var fileStream = file.OpenReadStream())
{
await blob.UploadAsync(fileStream, new BlobHttpHeaders { ContentType = file.ContentType },progressHandler:progressHandler);
}
Response.StatusCode = 400;
}
catch (Exception e)
{
Response.Clear();
Response.StatusCode = 204;
Response.HttpContext.Features.Get<IHttpResponseFeature>().ReasonPhrase = "File failed to upload";
Response.HttpContext.Features.Get<IHttpResponseFeature>().ReasonPhrase = e.Message;
}
}
private double GetProgressPercentage(double totalSize, double currentSize)
{
return (currentSize / totalSize) * 100;
}
private void UploadProgressChanged(object sender, long bytesUploaded)
{
uploadPercentage = GetProgressPercentage(uploadFileSize, bytesUploaded);
}
I am posting this file and it does upload but the file upload progress event is inaccurate it says the file upload is complete after a few seconds when in reality the file takes ~90 secs on my connection to appear in the Azure Blob Storage container.
So in the code above i have the progress handler which works (I can put a break point on it and see it increasing) but how do I return this value to the UI?
I found one solution that used Microsoft.AspNetCore.SignalR but I can't manage to integrate this into my own code and I'm not even sure if I'm on the right track.
using BlazorReportProgress.Server.Hubs;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.SignalR;
using System.Threading;
namespace BlazorReportProgress.Server.Controllers
{
[ApiController]
[Route("api/[controller]")]
public class SlowProcessController : ControllerBase
{
private readonly ILogger<SlowProcessController> _logger;
private readonly IHubContext<ProgressHub> _hubController;
public SlowProcessController(
ILogger<SlowProcessController> logger,
IHubContext<ProgressHub> hubContext)
{
_logger = logger;
_hubController = hubContext;
}
[HttpGet("{ClientID}")]
public IEnumerable<int> Get(string ClientID)
{
List<int> retVal = new();
_logger.LogInformation("Incoming call from ClientID : {ClientID}", ClientID);
_hubController.Clients.Client(ClientID).SendAsync("ProgressReport", "Starting...");
Thread.Sleep(1000);
for (int loop = 0; loop < 10; loop++)
{
_hubController.Clients.Client(ClientID).SendAsync("ProgressReport", loop.ToString());
retVal.Add(loop);
Thread.Sleep(500);
}
_hubController.Clients.Client(ClientID).SendAsync("ProgressReport", "Done!");
return retVal;
}
}
}
I read the Steve Sandersen blog but this says not to use the code as its been superceded by inbuilt blazor functionality.
My application is only for a few users and so i'm not too worried about backend APIs etc, If the upload component used a service not a controller I could more easily get the progress, but the compoents all seem to post to controllers.
Can anyone please enlighten me as to the best way to solve this?
So I have written a simple Azure Function (AF) that accepts (via Http Post method) an IFormCollection, loops through the file collection, pushes each file into an Azure Blob storage container and returns the url to each file.
The function itself works perfectly when I do a single file or multiple file post through Postman using the 'multipart/form-data' header. However when I try to post a file through an xUnit test, I get the following error:
System.IO.InvalidDataException : Multipart body length limit 16384 exceeded.
I have searched high and low for a solution, tried different things, namely;
Replicating the request object to be as close as possible to Postmans request.
Playing around with the 'boundary' in the header.
Setting 'RequestFormLimits' on the function.
None of these have helped so far.
The details are the project are as follows:
Azure Function v3: targeting .netcoreapp3.1
Startup.cs
public class Startup : FunctionsStartup
{
public IConfiguration Configuration { get; private set; }
public override void Configure(IFunctionsHostBuilder builder)
{
var x = builder;
InitializeConfiguration(builder);
builder.Services.AddSingleton(Configuration.Get<UploadImagesAppSettings>());
builder.Services.AddLogging();
builder.Services.AddSingleton<IBlobService,BlobService>();
}
private void InitializeConfiguration(IFunctionsHostBuilder builder)
{
var executionContextOptions = builder
.Services
.BuildServiceProvider()
.GetService<IOptions<ExecutionContextOptions>>()
.Value;
Configuration = new ConfigurationBuilder()
.SetBasePath(executionContextOptions.AppDirectory)
.AddJsonFile("appsettings.json")
.AddJsonFile("appsettings.Development.json", optional: true)
.AddEnvironmentVariables()
.Build();
}
}
UploadImages.cs
public class UploadImages
{
private readonly IBlobService BlobService;
public UploadImages(IBlobService blobService)
{
BlobService = blobService;
}
[FunctionName("UploadImages")]
[RequestFormLimits(ValueLengthLimit = int.MaxValue,
MultipartBodyLengthLimit = 60000000, ValueCountLimit = 10)]
public async Task<IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "images")] HttpRequest req)
{
List<Uri> returnUris = new List<Uri>();
if (req.ContentLength == 0)
{
string badResponseMessage = $"Request has no content";
return new BadRequestObjectResult(badResponseMessage);
}
if (req.ContentType.Contains("multipart/form-data") && req.Form.Files.Count > 0)
{
foreach (var file in req.Form.Files)
{
if (!file.IsValidImage())
{
string badResponseMessage = $"{file.FileName} is not a valid/accepted Image file";
return new BadRequestObjectResult(badResponseMessage);
}
var uri = await BlobService.CreateBlobAsync(file);
if (uri == null)
{
return new ObjectResult($"Could not blob the file {file.FileName}.");
}
returnUris.Add(uri);
}
}
if (!returnUris.Any())
{
return new NoContentResult();
}
return new OkObjectResult(returnUris);
}
}
Exception Thrown:
The below exception is thrown at the second if statement above, when it tries to process req.Form.Files.Count > 0, i.e.
if (req.ContentType.Contains("multipart/form-data") && req.Form.Files.Count > 0) {}
Message:
System.IO.InvalidDataException : Multipart body length limit 16384 exceeded.
Stack Trace:
MultipartReaderStream.UpdatePosition(Int32 read)
MultipartReaderStream.ReadAsync(Byte[] buffer, Int32 offset, Int32 count, CancellationToken cancellationToken)
StreamHelperExtensions.DrainAsync(Stream stream, ArrayPool`1 bytePool, Nullable`1 limit, CancellationToken cancellationToken)
MultipartReader.ReadNextSectionAsync(CancellationToken cancellationToken)
FormFeature.InnerReadFormAsync(CancellationToken cancellationToken)
FormFeature.ReadForm()
DefaultHttpRequest.get_Form()
UploadImages.Run(HttpRequest req) line 42
UploadImagesTests.HttpTrigger_ShouldReturnListOfUploadedUris(String fileNames)
xUnit Test Project: targeting .netcoreapp3.1
Over to the xUnit Test project, basically I am trying to write an integration test. The project references the AF project and has the following classes:
TestHost.cs
public class TestHost
{
public TestHost()
{
var startup = new TestStartup();
var host = new HostBuilder()
.ConfigureWebJobs(startup.Configure)
.ConfigureServices(ReplaceTestOverrides)
.Build();
ServiceProvider = host.Services;
}
public IServiceProvider ServiceProvider { get; }
private void ReplaceTestOverrides(IServiceCollection services)
{
// services.Replace(new ServiceDescriptor(typeof(ServiceToReplace), testImplementation));
}
private class TestStartup : Startup
{
public override void Configure(IFunctionsHostBuilder builder)
{
SetExecutionContextOptions(builder);
base.Configure(builder);
}
private static void SetExecutionContextOptions(IFunctionsHostBuilder builder)
{
builder.Services.Configure<ExecutionContextOptions>(o => o.AppDirectory = Directory.GetCurrentDirectory());
}
}
}
TestCollection.cs
[CollectionDefinition(Name)]
public class TestCollection : ICollectionFixture<TestHost>
{
public const string Name = nameof(TestCollection);
}
HttpRequestFactory.cs: To create Http Post Request
public static class HttpRequestFactory
{
public static DefaultHttpRequest Create(string method, string contentType, Stream body)
{
var request = new DefaultHttpRequest(new DefaultHttpContext());
var contentTypeWithBoundary = new MediaTypeHeaderValue(contentType)
{
Boundary = $"----------------------------{DateTime.Now.Ticks.ToString("x")}"
};
var boundary = MultipartRequestHelper.GetBoundary(
contentTypeWithBoundary, (int)body.Length);
request.Method = method;
request.Headers.Add("Cache-Control", "no-cache");
request.Headers.Add("Content-Type", contentType);
request.ContentType = $"{contentType}; boundary={boundary}";
request.ContentLength = body.Length;
request.Body = body;
return request;
}
private static string GetBoundary(MediaTypeHeaderValue contentType, int lengthLimit)
{
var boundary = HeaderUtilities.RemoveQuotes(contentType.Boundary);
if (string.IsNullOrWhiteSpace(boundary.Value))
{
throw new InvalidDataException("Missing content-type boundary.");
}
if (boundary.Length > lengthLimit)
{
throw new InvalidDataException(
$"Multipart boundary length limit {lengthLimit} exceeded.");
}
return boundary.Value;
}
}
The MultipartRequestHelper.cs class is available here
And Finally the Test class:
[Collection(TestCollection.Name)]
public class UploadImagesTests
{
readonly UploadImages UploadImagesFunction;
public UploadImagesTests(TestHost testHost)
{
UploadImagesFunction = new UploadImages(testHost.ServiceProvider.GetRequiredService<IBlobService>());
}
[Theory]
[InlineData("testfile2.jpg")]
public async void HttpTrigger_ShouldReturnListOfUploadedUris(string fileNames)
{
var formFile = GetFormFile(fileNames);
var fileStream = formFile.OpenReadStream();
var request = HttpRequestFactory.Create("POST", "multipart/form-data", fileStream);
var response = (OkObjectResult)await UploadImagesFunction.Run(request);
//fileStream.Close();
Assert.True(response.StatusCode == StatusCodes.Status200OK);
}
private static IFormFile GetFormFile(string fileName)
{
string fileExtension = fileName.Substring(fileName.IndexOf('.') + 1);
string fileNameandPath = GetFilePathWithName(fileName);
IFormFile formFile;
var stream = File.OpenRead(fileNameandPath);
switch (fileExtension)
{
case "jpg":
formFile = new FormFile(stream, 0, stream.Length,
fileName.Substring(0, fileName.IndexOf('.')),
fileName)
{
Headers = new HeaderDictionary(),
ContentType = "image/jpeg"
};
break;
case "png":
formFile = new FormFile(stream, 0, stream.Length,
fileName.Substring(0, fileName.IndexOf('.')),
fileName)
{
Headers = new HeaderDictionary(),
ContentType = "image/png"
};
break;
case "pdf":
formFile = new FormFile(stream, 0, stream.Length,
fileName.Substring(0, fileName.IndexOf('.')),
fileName)
{
Headers = new HeaderDictionary(),
ContentType = "application/pdf"
};
break;
default:
formFile = null;
break;
}
return formFile;
}
private static string GetFilePathWithName(string filename)
{
var outputFolder = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
return $"{outputFolder.Substring(0, outputFolder.IndexOf("bin"))}testfiles\\{filename}";
}
}
The test seems to be hitting the function and req.ContentLength does have a value. Considering this, could it have something to do with the way the File Streams are being managed? Perhaps not the right way?
Any inputs on this would be greatly appreciated.
Thanks
UPDATE 1
As per this post, I have also tried setting the ValueLengthLimit and MultipartBodyLengthLimit in the Startup of the Azure Function and/or the Test Project as opposed to attributes on the Azure Function. The exception then changed to:
"The inner stream position has changed unexpectedly"
Following this, I then set the fileStream position in the test project to SeekOrigin.Begin. I started getting the same error:
"Multipart body length limit 16384 exceeded."
It took me a 50km bike ride and a good nights sleep but I finally figured this one out :-).
The Azure function (AF) accepts an HttpRequest object as a parameter with the name of 'req' i.e.
public async Task Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = "images")] HttpRequest req)
The hierarchy of the files object in the HttpRequest object (along with the parameter names) is as follows:
HttpRequest -> req
FormCollection -> Form
FormFileCollection -> Files
This is what the AF accepts and one would access the files collection by using req.Form.Files
In my test case, instead of posting a FormCollection object, I was trying to post a Stream of a file to the Azure Function.
var formFile = GetFormFile(fileNames);
var fileStream = formFile.OpenReadStream();
var request = HttpRequestFactory.Create("POST", "multipart/form-data", fileStream);
As a result of this, req.Form had a Stream value that it could not interpret and the req.Form.Files was raising an exception.
In order to rectify this, I had to do the following:
Revert all changes made as part of UPDATE 1. This means that I removed the 'RequestFormLimits' settings from the Startup file and left them as attributes on the functions Run method.
Instantiate a FormFileCollection object and add the IFormFile to it
Instantiate a FormCollection object using this FormFileCollection as a parameter.
Add the FormCollection to the request object.
To achieve the above, I had to make the following changes in code.
Change 'Create' method in the HttpRequestFactory
public static DefaultHttpRequest Create(string method, string contentType, FormCollection formCollection)
{
var request = new DefaultHttpRequest(new DefaultHttpContext());
var boundary = $"----------------------------{DateTime.Now.Ticks.ToString("x")}";
request.Method = method;
request.Headers.Add("Cache-Control", "no-cache");
request.Headers.Add("Content-Type", contentType);
request.ContentType = $"{contentType}; boundary={boundary}";
request.Form = formCollection;
return request;
}
Add a private static GetFormFiles() method
I wrote an additional GetFormFiles() method that calls the existing GetFormFile() method, instantiate a FormFileCollection object and add the IFormFile to it. This method in turn returns a FormFileCollection.
private static FormFileCollection GetFormFiles(string fileNames)
{
var formFileCollection = new FormFileCollection();
foreach (var file in fileNames.Split(','))
{
formFileCollection.Add(GetFormFile(file));
}
return formFileCollection;
}
Change the Testmethod
The test method calls the GetFormFiles() to get a FormFileCollection then
instantiates a FormCollection object using this FormFileCollection as a parameter and then passes the FormCollection object as a parameter to the HttpRequest object instead of passing a Stream.
[Theory]
[InlineData("testfile2.jpg")]
public async void HttpTrigger_ShouldReturnListOfUploadedUris(string fileNames)
{
var formFiles = GetFormFiles(fileNames);
var formCollection = new FormCollection(null, formFiles);
var request = HttpRequestFactory.Create("POST", "multipart/form-data", formCollection);
var response = (OkObjectResult) await UploadImagesFunction.Run(request);
Assert.True(response.StatusCode == StatusCodes.Status200OK);
}
So in the end the issue was not really with the 'RequestFormLimits' but rather with the type of data I was submitting in the POST message.
I hope this answer provides a different perspective to someone that comes across the same error message.
Cheers.
I have a asp.net core app that uploads file to azure blob storage. The upload completes successfully but when CloudBlobContainer calls CreateIfNotExistAsync the web page crashes with a "The container already exists error".
var container = BlobClient.GetContainerReference(containerName.ToString().ToLower());
await container.CreateIfNotExistsAsync();
return container;
I have tried using surronding CreateIfNotExistsAsync() with the following if
if (! await container.ExistsAsync())
but that still errors.
The container name and the AccountName= in the connection string is lowercase.
I am using the latest stable Microsoft.Azure.Storage.Blob NuGet package 10.0.2
I have tried to catch the StroageExeception but the exception is not called
try
{
await container.CreateIfNotExistsAsync();
}
catch (StorageException ex)
{
Logger.LogError(ex.ToString());
}
I have been through all the points in this previous question but none of them apply/work in my scenario Azure Blob 400 Bad request on Creation of container
public class CloudBlobStorageProvider : ICloudBlobStorageProvider
{
private CloudBlobClient BlobClient { get; }
private ILogger Logger { get; }
public CloudBlobStorageProvider(IConfiguration configuration, ILogger<CloudBlobStorageProvider> logger)
{
var connectionString = configuration.GetConnectionString("AzureStorageAccount");
if (!CloudStorageAccount.TryParse(connectionString, out CloudStorageAccount storageAccount))
{
logger.LogError($"The supplied connection string for Azure blob storage could not be parsed: {connectionString}");
}
BlobClient = storageAccount.CreateCloudBlobClient();
}
public async Task<CloudBlobContainer> GetContainerAsync(CloudBlobContainerName containerName)
{
var container = BlobClient.GetContainerReference(containerName.ToString().ToLower());
await container.CreateIfNotExistsAsync(BlobContainerPublicAccessType.Off, null, null);
return container;
}
}
public interface ICloudBlobStorageProvider
{
Task<CloudBlobContainer> GetContainerAsync(CloudBlobContainerName containerName);
}
Which is called by
public async Task<CloudBlockBlob> UploadServiceUserAttachmentAsync(IFormFile formFile)
{
var fileExtenion = RegularExpressionHelpers.GetFileExtension(formFile.FileName);
string attachmentFileName = (string.IsNullOrEmpty(fileExtenion)) ? $"{Guid.NewGuid().ToString()}" : $"{Guid.NewGuid().ToString()}{fileExtenion}";
var userAttachmentContainer = await CloudBlobStorageProvider.GetContainerAsync(CloudBlobContainerName.userattachments);
var blobBlockReference = userAttachmentContainer.GetBlockBlobReference(attachmentFileName);
try
{
using (var stream = formFile.OpenReadStream())
{
await blobBlockReference.UploadFromStreamAsync(stream);
await blobBlockReference.FetchAttributesAsync();
var blobProperties = blobBlockReference.Properties;
blobProperties.ContentType = formFile.ContentType;
await blobBlockReference.SetPropertiesAsync();
}
}
catch (Exception e)
{
Logger.LogWarning(e, $"Exception encountered while attempting to write profile photo to blob storage");
}
return blobBlockReference;
}
I have an Azure job which asynchronously saves records to the database. I am finding that it does not actually save anything to the database. I am definitely using async/awaits everywhere. I am adding parent (market) and child records. My database has referential constraints so the parent has to exist before the child, but that should be fine as I am doing them in the right order. I have no try-catches around my methods and there is nothing in the azure logs so it appears the job is succeeding. I have called Method1 with an await from a Winforms exe and it works fine from that. What can be wrong?
public static async Task MyJob([TimerTrigger("00:02:00", RunOnStartup = true)] TimerInfo timerInfo, TextWriter log)
{
await Jobs.Method1(Client, Logger);
}
public static async Task Method1(IClient client, ILogger logger)
{
await DataRepository.AddMarket(event.Id, event.MarketId);
await DataRepository.AddMarketChild(event.MarketId, 999);
}
public static async Task<Market> AddMarket(string eventId, string marketId)
{
using (var ctx = BTBEntities.CreateContext())
{
var market = new Market()
{
MarketId = marketId,
EventId = eventId,
};
ctx.Markets.Add(market);
await ctx.SaveChangesAsync();
return market;
}
}
public static async Task<HorseBet> AddHorseBet(string marketId, long selectionId)
{
using (var ctx = BTBEntities.CreateContext())
{
var bet = new MarketChild()
{
MarketId = marketId,
SelectionId = selectionId,
};
ctx.MarketChilds.Add(bet);
await ctx.SaveChangesAsync();
return bet;
}
}
Turns out there was a database problem it was just being swallowed by Azure. This is how I fixed it.
public override System.Threading.Tasks.Task<int> SaveChangesAsync()
{
try
{
return base.SaveChangesAsync();
}
catch (DbEntityValidationException ex)
{
var errorMessages = ex.EntityValidationErrors
.SelectMany(x => x.ValidationErrors)
.Select(x => x.ErrorMessage);
var fullErrorMessage = string.Join("; ", errorMessages);
var exceptionMessage = string.Concat(ex.Message, " The validation errors are: ", fullErrorMessage);
throw new DbEntityValidationException(exceptionMessage, ex.EntityValidationErrors);
}
}
I have an Azure Web Job built using the Azure SDK whose only job is to call a web service (Web API) and then log a response based on the return value (a class). The problem is that as soon as it calls the HttpClient PostAsJsonAsync method to call the service, it exits out of the web job without executing any of the response handling. My code is:
public class Result
{
// Properties ---------------------------------------------------------
public bool Success { get; set; }
public string Error { get; set; }
}
public class Functions
{
// This function will be triggered based on the schedule you have set for this WebJob
// This function will enqueue a message on an Azure Queue called queue
[NoAutomaticTrigger]
public async static void ManualTrigger(TextWriter log, int value)
{
using (var client = new HttpClient())
{
var rootUrl = ConfigurationManager.AppSettings.Get("WebJobTargetUrl");
client.BaseAddress = new System.Uri(rootUrl);
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
Console.WriteLine("Call service");
var response = await client.PostAsJsonAsync("api/Reminder/ProcessDueReminders", new { ItemID = 1 });
Console.WriteLine("After service");
var result = await response.Content.ReadAsAsync<Result>();
Console.WriteLine("After result");
if (result.Success)
Console.WriteLine("Reminders Processed");
else
Console.WriteLine("Reminder process error: " + result.Error);
}
}
}
and the execution logs from the portal are:
I believe it has something to do with the asynchronous operation but I can't figure out a pattern that will work. Any help would be appreciated.
You must define the return value of your own async method as Task instead of void.
On a related note, you should suffix the name of your method with Async. That's not going to solve the problem, but it indicates that you're using the async/await pattern.
There is probably an exception in your PostAsJsonAsync call. Try to put a try catch around it to and log the error:
try {
var response = await client.PostAsJsonAsync("api/Reminder/ProcessDueReminders", new { ItemID = 1 });
} catch (Exception ex){
Console.WriteLine("Exception: "+ ex);
}
Console.WriteLine("After service");