Zipping files and uploading to Azure Blobstorage using filestream - azure

I am creating a website with ASP.NET Core, and I need to take a bunch of files, zip them together and then upload them to Azure Blobstorage.
It succeeds, I get a downloadlink but when I download the file, I just get a "File" that I cannot open with anything.
Here's the code that handles the uploading:
var moduleVersionReleaseFiles = _appDbContext
.ModuleVersionReleaseFile
.Where(x => x.ModuleVersionReleaseId == moduleVersionReleaseId)
.ToList();
string AzureBlobStorageConfig = _configuration["AzureBlobStorage"];
var connectionString = _configuration["StorageConnectionString"];
string containerName = moduleVersionReleaseId + "-" + Guid.NewGuid().ToString();
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName.ToLowerInvariant(), Azure.Storage.Blobs.Models.PublicAccessType.BlobContainer);
BlobClient blobClient = containerClient.GetBlobClient(moduleVersionReleaseId.ToString());
using var stream = new MemoryStream();
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var moduleVersionReleaseFile in moduleVersionReleaseFiles)
{
var entry = archive.CreateEntry(moduleVersionReleaseFile.File);
using (var entryStream = entry.Open())
using (var fileStream = File.OpenRead($"{moduleVersionReleaseFile.Path}\\{moduleVersionReleaseFile.File}"))
{
await fileStream.CopyToAsync(entryStream);
}
}
stream.Position = 0;
await blobClient.UploadAsync(stream, true);
await stream.FlushAsync();
}
return blobClient.Uri.ToString();
I am using Azurite for testing, so I can't log onto the blobstorage and check there.
Why am I not getting a zip file when downloading the file as expected?

The reason you're not able to open the zip file is because it is not getting saved properly. Can you please try with the following code:
using var stream = new MemoryStream();
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var moduleVersionReleaseFile in moduleVersionReleaseFiles)
{
var entry = archive.CreateEntry(moduleVersionReleaseFile.File);
using (var entryStream = entry.Open())
var fileContents = File.ReadAllBytes($"{moduleVersionReleaseFile.Path}\\{moduleVersionReleaseFile.File}"));
using (BinaryWriter zipFileBinary = new BinaryWriter(entryStream))
{
zipFileBinary.Write(fileContents);
}
}
stream.Position = 0;
await blobClient.UploadAsync(stream, true);
}

Related

Migrate from Azure to SharePoint using Migration API - "Must specify IV in Manifest or blob metadata'"

I am trying to use the Migration API with C# to migrate from the default free Azure Cloud (given to all sharepoint accounts) to SharePoint.
I can successfully generate a cloud container, and upload files, but the migration fails. Here is my code:
private void MigrateToAzureThenSharePoint()
{
//Provision The default Azure Containers and Migation Queue
var migrationContainers = destinationCtx.Site.ProvisionMigrationContainers();
var migrationQueue = destinationCtx.Site.ProvisionMigrationQueue();
destinationCtx.ExecuteQuery();
var containerInfoList = migrationContainers.Value;
var dataContainerUri = containerInfoList.DataContainerUri;
var metadataContainerUri = containerInfoList.MetadataContainerUri;
var queueInfo = migrationQueue.Value;
//Instansiate Cloud Queue
var cloudQueue = new CloudQueue(new Uri(queueInfo.JobQueueUri));
//Instansiate CloudBlobContainer
CloudBlobContainer dataContainer = new CloudBlobContainer(new Uri(dataContainerUri));
CloudBlobContainer manifestContainer = new CloudBlobContainer(new Uri(metadataContainerUri));
//Test files to upload
var testfiles = new[]
{
new SourceFile
{
Filename = "test.txt",
LastModified = DateTime.Now,
Contents = Encoding.UTF8.GetBytes("Hi, this is a test text-file"),
Title = "Title of file 1"
},
new SourceFile
{
Filename = "test2.txt",
LastModified = DateTime.Now.AddDays(-1),
Contents = Encoding.UTF8.GetBytes("Tesfile2"),
Title = "Second title"
}
};
//Upload Test Files to Cloud
foreach (var testfile in testfiles)
{
var blobReference = dataContainer.GetBlockBlobReference(testfile.Filename);
blobReference.UploadFromByteArray(testfile.Contents, 0, testfile.Contents.Length, null);
blobReference.CreateSnapshot();
}
var manifestPackage = new ManifestPackage(destOrganisationList.Id,
destinationCtx.Web.Id,
destinationCtx.Web.Title,
destOrganisationList.Title,
#"\",
destOrganisationList.RootFolder.UniqueId,
destOrganisationList.RootFolder.ParentFolder.UniqueId
);
//Create XML Files for Azure Migration (An easier way here would be great? I manualy write them)
var filesInManifestPackage = manifestPackage.GetManifestPackageFiles(testfiles);
//Uploaded manually created XML files
foreach (var migrationPackageFile in filesInManifestPackage)
{
var blobReference = manifestContainer.GetBlockBlobReference(migrationPackageFile.Filename);
blobReference.UploadFromByteArray(migrationPackageFile.Contents, 0, migrationPackageFile.Contents.Length, null);
blobReference.CreateSnapshot();
}
//Start Migration
var result = destinationCtx.Site.CreateMigrationJobEncrypted(destinationCtx.Web.Id,
containerInfoList.DataContainerUri,
containerInfoList.MetadataContainerUri,
queueInfo.JobQueueUri,
new EncryptionOption()
{
AES256CBCKey = containerInfoList.EncryptionKey
});
destinationCtx.ExecuteQuery();
//Check for messags
while (true) {
var msg = cloudQueue.GetMessage();
if (msg == null)
{
Task.Delay(TimeSpan.FromSeconds(1));
continue;
}
var message = JsonConvert.DeserializeObject<EncryptedMessage>(msg.AsString);
cloudQueue.DeleteMessage(msg);
//Decode encrypted message
var decoded = JsonConvert.DeserializeObject<UpdateMessage>(Decrypt(message.Content, containerInfoList.EncryptionKey, message.IV));
//Error
}
}
There are a few errors, but they all are similar to this:
"Unable to download SystemData.xml with exception 'Must specify IV in
Manifest or blob metadata'"
I believe I may have to encrypt the files when I upload from this method:
blobReference.UploadFromByteArray
There is an option:
var blobRequestOptions = new BlobRequestOptions();
blobRequestOptions.EncryptionPolicy =
But I don't know how to instantiate the encryption policy (and I may be going down a rabbit hole)
Any help would be greatly appreciated, as documentation online is shocking minimal.

How to zip blobs using ICSharpCode.SharpZipLib.Zip

I am using latest the azure sdk Azure.Storage.Blobs version 12.9.1 for C# . I have to move all blobs from one container to another . I have listed all blobs but I am stuck at how to open an output stream to write a blob in another container .
FileStream fs = File.OpenWrite("\\" + str+"zip");
using (var zipOutputStream = new ZipOutputStream(fs))
{
foreach (var blobFileName in files)
{
zipOutputStream.SetLevel(5);
var blob = dirFromZip.GetBlobClient(blobFileName);
var entry = new ZipEntry(blobFileName);
zipOutputStream.PutNextEntry(entry);
blob.DownloadTo(zipOutputStream);
}
fs.Position = 0;
var blobClient = dirwhereZip.GetBlobClient(str);
blobClient.Upload(fs);
zipOutputStream.Finish();
zipOutputStream.Close();
}
Error : System.NotSupportedException: Stream does not support reading.
Please try the code below:
using System;
using System.Threading.Tasks;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System.IO;
using ICSharpCode.SharpZipLib.Zip;
namespace SO68566758
{
class Program
{
private const string connectionString = "DefaultEndpointsProtocol=https;AccountName=account-name;AccountKey=account-key";
private const string sourceContainer = "source-container";
private const string targetContainer = "target-container";
private const string outputBlobName = "backup.zip";
static async Task Main(string[] args)
{
BlobServiceClient serviceClient = new BlobServiceClient(connectionString);
BlobContainerClient sourceContainerClient = serviceClient.GetBlobContainerClient(sourceContainer);
BlobContainerClient targetContainerClient = serviceClient.GetBlobContainerClient(targetContainer);
var blobs = sourceContainerClient.GetBlobsAsync();
using (var fs = new FileStream(outputBlobName, FileMode.OpenOrCreate))
{
using (var zipOutputStream = new ZipOutputStream(fs))
{
await foreach (var blob in blobs)
{
var blobName = blob.Name;
var blobClient = sourceContainerClient.GetBlobClient(blobName);
//var downloadResponse = await blobClient.DownloadAsync();
//var streamContent = downloadResponse.Value.Content;
var entry = new ZipEntry(blobName);
zipOutputStream.PutNextEntry(entry);
await blobClient.DownloadToAsync(zipOutputStream);
}
}
}
BlobClient targetBlob = targetContainerClient.GetBlobClient(outputBlobName);
using (FileStream fs = new FileStream(outputBlobName, FileMode.Open))
{
await targetBlob.UploadAsync(fs);
}
}
}
}

How to unzip file in Azure File Share using Azure Function?

I have an Azure Storage Account with Azure File share. I want to extract zip archive file to another dir in file share using Azure functions. I wrote this code in C#:
CloudFileDirectory rootDirectory = cloudFileShare.GetRootDirectoryReference();
CloudFileDirectory output = rootDirectory.GetDirectoryReference("output");
CloudFile cloudFile = input.GetFileReference("archive1.zip");
using (var stream = await cloudFile.OpenReadAsync())
{
var file1 = new ZipArchive(stream);
foreach (var zipEntry in file1.Entries)
{
var file2 = output.GetFileReference(zipEntry.Name);
var fileStream = zipEntry.Open();
await file2.UploadFromStreamAsync(fileStream); //error is in this line
}
}
But I got the error:
System.Private.CoreLib: Exception while executing function: HttpTriggerExtract. Microsoft.WindowsAzure.Storage:
Operation is not valid due to the current state of the object.
How to fix this?
Edit: I fix the error using MemoryStream in addition, this code works:
foreach (var zipEntry in file1.Entries) {
var fsz = output.GetFileReference(zipEntry.Name);
using (var ms = new MemoryStream())
{
using (var fileStream = zipEntry.Open())
{
await fileStream.CopyToAsync(ms);
ms.Seek(0, SeekOrigin.Begin);
await fsz.UploadFromStreamAsync(ms);
}
}
Regarding the issue, please refer to the following code (I use package WindowsAzure.Storage 9.3.1 to do that)
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
CloudFileClient cloudFileClient = storageAccount.CreateCloudFileClient();
CloudFileShare cloudFileShare = cloudFileClient.GetShareReference("share2");
CloudFileDirectory rootDirectory = cloudFileShare.GetRootDirectoryReference();
CloudFileDirectory input = rootDirectory.GetDirectoryReference("input");
CloudFileDirectory output = rootDirectory.GetDirectoryReference("output");
CloudFile cloudFile = input.GetFileReference("sample.zip");
using (var stream = await cloudFile.OpenReadAsync())
using (var zipArchive = new ZipArchive(stream)) {
foreach (var entry in zipArchive.Entries)
{
if (entry.Length > 0) {
CloudFile extractedFile = output.GetFileReference(entry.Name);
using (var entryStream = entry.Open())
{
byte[] buffer = new byte[16 * 1024];
using (var ms = await extractedFile.OpenWriteAsync(entry.Length))
{
int read;
while ((read = await entryStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, read);
}
}
}
}
}
}
The above answer helped me for my problem.
With the new Azure Library (12.7.0) you have to code this way:
string srcDir = #"sourcePath";
string destDir = #"sourcePath\testStorageUnzip";
string srcFileName = "AzureStorageZip.zip";
string azureConnectionString = ConfigurationManager.AppSettings["beecloudfileshare_AzureStorageConnectionString"];
StorageSharedKeyCredential credential = BeeFileManager.GetAzureStorageKeyCredential(azureConnectionString);
Uri srcUri = new Uri("https:" + Path.Combine(srcDir, srcFileName).Replace("\\", "/"), UriKind.Absolute);
Uri destDirUri = new Uri("https:" + Path.Combine(destDir).Replace("\\", "/"), UriKind.Absolute);
// Get a reference to the file we created previously
ShareFileClient sourceFile = new ShareFileClient(srcUri, credential);
ShareDirectoryClient shareDirectoryClient = new ShareDirectoryClient(destDirUri,credential);
shareDirectoryClient.CreateIfNotExistsAsync().GetAwaiter().GetResult();
using (var stream = sourceFile.OpenRead())
using (var zipArchive = new ZipArchive(stream))
{
foreach (var entry in zipArchive.Entries)
{
if (entry.Length > 0)
{
//CloudFile extractedFile = output.GetFileReference(entry.Name);
Uri destUri = new Uri("https:" + Path.Combine(destDir, entry.Name).Replace("\\", "/"), UriKind.Absolute);
ShareFileClient extractedFile = new ShareFileClient(destUri, credential);
using (var entryStream = entry.Open())
{
using (MemoryStream ms = new MemoryStream())
{
entryStream.CopyTo(ms);
//
//Sorry I have this part in another method
//
Uri fileUri = new Uri("https:" + Path.GetDirectoryName(filePath).Replace("\\", "/"), UriKind.Absolute);
// Get a reference to the file we created previously
ShareDirectoryClient directory = new ShareDirectoryClient(fileUri, credential);
ShareFileClient file = directory.GetFileClient(Path.GetFileName(filePath));
ms.Seek(0, SeekOrigin.Begin);
file.Create(ms.Length);
file.Upload(ms);
//
//
//
}
}
}
}
}
}

How can I dynamically create Ziparchive in Azure

I have multiple files in my Azure storage account say its Master container, I have created a dynamic container which will have the required files copied from Master Container, and those coped files needed to be zipped inside that created container. See below code, where in I have created the zip archive. Also the zip archive is getting created, but when I download manually and see the files, it seems corrupted (for ex. the actual size of the individual files are more than 1Mb but the files which I get after download seems 22Kb), and File formats would be .ipt, .iam (Autodesk Inventor Files)
CloudBlobContainer destContainer = blobClient.GetContainerReference(AzureContainer);
bool isCreated = destContainer.CreateIfNotExists();
var zipblob = destContainer.GetBlockBlobReference("inputAssembly.zip");
using (var stream = await zipblob.OpenWriteAsync())
using (var zip = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var fileName in inputfile)
{
using (var fileStream = new MemoryStream())
{
if (destContainer.GetBlockBlobReference(fileName).Exists())
{
destContainer.GetBlockBlobReference(fileName).DownloadToStream(fileStream);
}
var newZip = new ZipArchive(fileStream, ZipArchiveMode.Create);
var entry = newZip.CreateEntry(fileName, CompressionLevel.NoCompression);
using (var innerFile = entry.Open())
{
fileStream.CopyTo(innerFile);
}
fileStream.Close();
}
}
}
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(fixedpartContainer);
CloudBlobContainer destContainer1 = blobClient.GetContainerReference(AzureContainer);
bool isCreated = destContainer1.CreateIfNotExists();
var zipblob = destContainer1.GetBlockBlobReference("inputAssembly.zip");
using (var stream = await zipblob.OpenWriteAsync())
{
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var blobName in blobFileNames)
{
var blob = destContainer.GetBlockBlobClient(blobName);
var zipEntry = zipArchive.CreateEntry(blobName);
using var zipStream = zipEntry.Open();
using var fileStream = new MemoryStream();
await blob.DownloadToAsync(fileStream);
await zipStream.WriteAsync(fileStream.ToArray());
AssemblyCreated = true;
}
}
}

Email Attachments missing on Azure, works locally

I have implemented something similar to the following question and I can get it working locally on my server, but when I deploy to Azure it doesn't work. I don't get any errors: just an email without the attachment.
Sending attachments using Azure blobs
Are there restrictions on what types of files can be sent with SendGrid (file is only 56k)?
Does the Azure App service have to be at a particular level or can it be done on Basic?
The blob URL definitley exists and I am setting the stream to zero as suggested in that previous question.
MailMessage mm = new MailMessage("receiver address", "someone");
mm.From = new MailAddress("myAddress", "My Name");
mm.Subject = content.Subject;
mm.Body = content.Body;
mm.IsBodyHtml = true;
mm.BodyEncoding = UTF8Encoding.UTF8;
mm.DeliveryNotificationOptions = DeliveryNotificationOptions.OnFailure;
var attachmentAsStream = _storageAccessService.GetAssetAsStreamForEmail("myContainer", "fileThatExists.pdf");
var attachment = new Attachment(attachmentAsStream, "File.pdf", MediaTypeNames.Application.Pdf);
mm.Attachments.Add(attachment);
public MemoryStream GetAssetAsStreamForEmail(string containerName, string fileName)
{
// Create the blob client.
CloudBlobClient blobClient = StorageAccountReference().CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
var memoryStream = new MemoryStream();
try
{
using (var stream = new MemoryStream())
{
blob.DownloadToStream(memoryStream);
memoryStream.Seek(0, SeekOrigin.Begin);
}
}
catch (Exception ex)
{
Elmah.ErrorSignal.FromCurrentContext().Raise(new Exception("Failed to download Email Atatchment: "+ ex.Message));
}
memoryStream.Position = 0;
return memoryStream;
}
using (SmtpClient client = new SmtpClient())
{
client.Port = 587;
client.Host = "smtp.sendgrid.net";
client.EnableSsl = true;
client.Timeout = 10000;
client.DeliveryMethod = SmtpDeliveryMethod.Network;
client.UseDefaultCredentials = false;
client.Credentials = new System.Net.NetworkCredential("hidden", "hidden");
await client.SendMailAsync(message);
}
Update
Update after Yasir's suggestion below. Downloading the blob from Azure as a Stream only seems to work locally. But if I change to download as a ByteArray then it works everywhere, nonetheless...
public MemoryStream GetAssetAsStreamForEmail(string containerName, string fileName)
{
// Create the blob client.
CloudBlobClient blobClient = StorageAccountReference().CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
try
{
blob.FetchAttributes();
var fileStream = new byte[blob.Properties.Length];
for (int i = 0; i < blob.Properties.Length; i++)
{
fileStream[i] = 0x20;
}
blob.DownloadToByteArray(fileStream, 0) ;
MemoryStream bufferStream = new MemoryStream(fileStream);
}
catch (Exception ex)
{
Elmah.ErrorSignal.FromCurrentContext().Raise(new Exception("Failed to download Email Atatchment: " + ex.Message));
}
return null;
}
The following code works for me for sending emails using Sendgrid from Azure functions. I attach a CSV file from Blob Storage. It should work for you as well. All you will have to do is ensure that you read the pdf file as a byte[].
public interface IEmailAttachment
{
string Name { get; }
byte[] FileData { get; }
}
public static void Send(MailMessage mailMessage, IEnumerable<IEmailAttachment> attachments)
{
try
{
// Get the configuration data
string server = ConfigReader.EmailServer;
int port = ConfigReader.EmailPort;
string username = ConfigReader.SendGridUserName;
string password = ConfigReader.SendGridPassword;
smtpClient.EnableSsl = false;
smtpClient.Credentials = new NetworkCredential(username, password);
// Create the SMTP Client
SmtpClient smtpClient = new SmtpClient(server, port);
// Prepare the MailMessage
mailMessage.From = new MailAddress(ConfigReader.FromEmail);
var toEmails = ConfigReader.ToEmail.Split(',');
foreach (var toEmail in toEmails)
{
mailMessage.To.Add(toEmail);
}
var ccEmails = ConfigReader.EmailCc.Split(',');
foreach (var ccEmail in ccEmails)
{
mailMessage.CC.Add(ccEmail);
}
// Add attachments
List<MemoryStream> files = new List<MemoryStream>();
if (attachments != null)
{
foreach (IEmailAttachment file in attachments)
{
MemoryStream bufferStream = new MemoryStream(file.FileData);
files.Add(bufferStream);
Attachment attachment = new Attachment(bufferStream, file.Name);
mailMessage.Attachments.Add(attachment);
}
}
mailMessage.IsBodyHtml = true;
// Send the email
smtpClient.Send(mailMessage);
foreach (MemoryStream stream in files)
{
stream.Dispose();
}
}
catch (Exception)
{
throw;
}
}

Resources