How to unzip file in Azure File Share using Azure Function? - azure

I have an Azure Storage Account with Azure File share. I want to extract zip archive file to another dir in file share using Azure functions. I wrote this code in C#:
CloudFileDirectory rootDirectory = cloudFileShare.GetRootDirectoryReference();
CloudFileDirectory output = rootDirectory.GetDirectoryReference("output");
CloudFile cloudFile = input.GetFileReference("archive1.zip");
using (var stream = await cloudFile.OpenReadAsync())
{
var file1 = new ZipArchive(stream);
foreach (var zipEntry in file1.Entries)
{
var file2 = output.GetFileReference(zipEntry.Name);
var fileStream = zipEntry.Open();
await file2.UploadFromStreamAsync(fileStream); //error is in this line
}
}
But I got the error:
System.Private.CoreLib: Exception while executing function: HttpTriggerExtract. Microsoft.WindowsAzure.Storage:
Operation is not valid due to the current state of the object.
How to fix this?
Edit: I fix the error using MemoryStream in addition, this code works:
foreach (var zipEntry in file1.Entries) {
var fsz = output.GetFileReference(zipEntry.Name);
using (var ms = new MemoryStream())
{
using (var fileStream = zipEntry.Open())
{
await fileStream.CopyToAsync(ms);
ms.Seek(0, SeekOrigin.Begin);
await fsz.UploadFromStreamAsync(ms);
}
}

Regarding the issue, please refer to the following code (I use package WindowsAzure.Storage 9.3.1 to do that)
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
CloudFileClient cloudFileClient = storageAccount.CreateCloudFileClient();
CloudFileShare cloudFileShare = cloudFileClient.GetShareReference("share2");
CloudFileDirectory rootDirectory = cloudFileShare.GetRootDirectoryReference();
CloudFileDirectory input = rootDirectory.GetDirectoryReference("input");
CloudFileDirectory output = rootDirectory.GetDirectoryReference("output");
CloudFile cloudFile = input.GetFileReference("sample.zip");
using (var stream = await cloudFile.OpenReadAsync())
using (var zipArchive = new ZipArchive(stream)) {
foreach (var entry in zipArchive.Entries)
{
if (entry.Length > 0) {
CloudFile extractedFile = output.GetFileReference(entry.Name);
using (var entryStream = entry.Open())
{
byte[] buffer = new byte[16 * 1024];
using (var ms = await extractedFile.OpenWriteAsync(entry.Length))
{
int read;
while ((read = await entryStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, read);
}
}
}
}
}
}

The above answer helped me for my problem.
With the new Azure Library (12.7.0) you have to code this way:
string srcDir = #"sourcePath";
string destDir = #"sourcePath\testStorageUnzip";
string srcFileName = "AzureStorageZip.zip";
string azureConnectionString = ConfigurationManager.AppSettings["beecloudfileshare_AzureStorageConnectionString"];
StorageSharedKeyCredential credential = BeeFileManager.GetAzureStorageKeyCredential(azureConnectionString);
Uri srcUri = new Uri("https:" + Path.Combine(srcDir, srcFileName).Replace("\\", "/"), UriKind.Absolute);
Uri destDirUri = new Uri("https:" + Path.Combine(destDir).Replace("\\", "/"), UriKind.Absolute);
// Get a reference to the file we created previously
ShareFileClient sourceFile = new ShareFileClient(srcUri, credential);
ShareDirectoryClient shareDirectoryClient = new ShareDirectoryClient(destDirUri,credential);
shareDirectoryClient.CreateIfNotExistsAsync().GetAwaiter().GetResult();
using (var stream = sourceFile.OpenRead())
using (var zipArchive = new ZipArchive(stream))
{
foreach (var entry in zipArchive.Entries)
{
if (entry.Length > 0)
{
//CloudFile extractedFile = output.GetFileReference(entry.Name);
Uri destUri = new Uri("https:" + Path.Combine(destDir, entry.Name).Replace("\\", "/"), UriKind.Absolute);
ShareFileClient extractedFile = new ShareFileClient(destUri, credential);
using (var entryStream = entry.Open())
{
using (MemoryStream ms = new MemoryStream())
{
entryStream.CopyTo(ms);
//
//Sorry I have this part in another method
//
Uri fileUri = new Uri("https:" + Path.GetDirectoryName(filePath).Replace("\\", "/"), UriKind.Absolute);
// Get a reference to the file we created previously
ShareDirectoryClient directory = new ShareDirectoryClient(fileUri, credential);
ShareFileClient file = directory.GetFileClient(Path.GetFileName(filePath));
ms.Seek(0, SeekOrigin.Begin);
file.Create(ms.Length);
file.Upload(ms);
//
//
//
}
}
}
}
}
}

Related

Zipping files and uploading to Azure Blobstorage using filestream

I am creating a website with ASP.NET Core, and I need to take a bunch of files, zip them together and then upload them to Azure Blobstorage.
It succeeds, I get a downloadlink but when I download the file, I just get a "File" that I cannot open with anything.
Here's the code that handles the uploading:
var moduleVersionReleaseFiles = _appDbContext
.ModuleVersionReleaseFile
.Where(x => x.ModuleVersionReleaseId == moduleVersionReleaseId)
.ToList();
string AzureBlobStorageConfig = _configuration["AzureBlobStorage"];
var connectionString = _configuration["StorageConnectionString"];
string containerName = moduleVersionReleaseId + "-" + Guid.NewGuid().ToString();
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName.ToLowerInvariant(), Azure.Storage.Blobs.Models.PublicAccessType.BlobContainer);
BlobClient blobClient = containerClient.GetBlobClient(moduleVersionReleaseId.ToString());
using var stream = new MemoryStream();
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var moduleVersionReleaseFile in moduleVersionReleaseFiles)
{
var entry = archive.CreateEntry(moduleVersionReleaseFile.File);
using (var entryStream = entry.Open())
using (var fileStream = File.OpenRead($"{moduleVersionReleaseFile.Path}\\{moduleVersionReleaseFile.File}"))
{
await fileStream.CopyToAsync(entryStream);
}
}
stream.Position = 0;
await blobClient.UploadAsync(stream, true);
await stream.FlushAsync();
}
return blobClient.Uri.ToString();
I am using Azurite for testing, so I can't log onto the blobstorage and check there.
Why am I not getting a zip file when downloading the file as expected?
The reason you're not able to open the zip file is because it is not getting saved properly. Can you please try with the following code:
using var stream = new MemoryStream();
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var moduleVersionReleaseFile in moduleVersionReleaseFiles)
{
var entry = archive.CreateEntry(moduleVersionReleaseFile.File);
using (var entryStream = entry.Open())
var fileContents = File.ReadAllBytes($"{moduleVersionReleaseFile.Path}\\{moduleVersionReleaseFile.File}"));
using (BinaryWriter zipFileBinary = new BinaryWriter(entryStream))
{
zipFileBinary.Write(fileContents);
}
}
stream.Position = 0;
await blobClient.UploadAsync(stream, true);
}

How to zip blobs using ICSharpCode.SharpZipLib.Zip

I am using latest the azure sdk Azure.Storage.Blobs version 12.9.1 for C# . I have to move all blobs from one container to another . I have listed all blobs but I am stuck at how to open an output stream to write a blob in another container .
FileStream fs = File.OpenWrite("\\" + str+"zip");
using (var zipOutputStream = new ZipOutputStream(fs))
{
foreach (var blobFileName in files)
{
zipOutputStream.SetLevel(5);
var blob = dirFromZip.GetBlobClient(blobFileName);
var entry = new ZipEntry(blobFileName);
zipOutputStream.PutNextEntry(entry);
blob.DownloadTo(zipOutputStream);
}
fs.Position = 0;
var blobClient = dirwhereZip.GetBlobClient(str);
blobClient.Upload(fs);
zipOutputStream.Finish();
zipOutputStream.Close();
}
Error : System.NotSupportedException: Stream does not support reading.
Please try the code below:
using System;
using System.Threading.Tasks;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System.IO;
using ICSharpCode.SharpZipLib.Zip;
namespace SO68566758
{
class Program
{
private const string connectionString = "DefaultEndpointsProtocol=https;AccountName=account-name;AccountKey=account-key";
private const string sourceContainer = "source-container";
private const string targetContainer = "target-container";
private const string outputBlobName = "backup.zip";
static async Task Main(string[] args)
{
BlobServiceClient serviceClient = new BlobServiceClient(connectionString);
BlobContainerClient sourceContainerClient = serviceClient.GetBlobContainerClient(sourceContainer);
BlobContainerClient targetContainerClient = serviceClient.GetBlobContainerClient(targetContainer);
var blobs = sourceContainerClient.GetBlobsAsync();
using (var fs = new FileStream(outputBlobName, FileMode.OpenOrCreate))
{
using (var zipOutputStream = new ZipOutputStream(fs))
{
await foreach (var blob in blobs)
{
var blobName = blob.Name;
var blobClient = sourceContainerClient.GetBlobClient(blobName);
//var downloadResponse = await blobClient.DownloadAsync();
//var streamContent = downloadResponse.Value.Content;
var entry = new ZipEntry(blobName);
zipOutputStream.PutNextEntry(entry);
await blobClient.DownloadToAsync(zipOutputStream);
}
}
}
BlobClient targetBlob = targetContainerClient.GetBlobClient(outputBlobName);
using (FileStream fs = new FileStream(outputBlobName, FileMode.Open))
{
await targetBlob.UploadAsync(fs);
}
}
}
}

How can I dynamically create Ziparchive in Azure

I have multiple files in my Azure storage account say its Master container, I have created a dynamic container which will have the required files copied from Master Container, and those coped files needed to be zipped inside that created container. See below code, where in I have created the zip archive. Also the zip archive is getting created, but when I download manually and see the files, it seems corrupted (for ex. the actual size of the individual files are more than 1Mb but the files which I get after download seems 22Kb), and File formats would be .ipt, .iam (Autodesk Inventor Files)
CloudBlobContainer destContainer = blobClient.GetContainerReference(AzureContainer);
bool isCreated = destContainer.CreateIfNotExists();
var zipblob = destContainer.GetBlockBlobReference("inputAssembly.zip");
using (var stream = await zipblob.OpenWriteAsync())
using (var zip = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var fileName in inputfile)
{
using (var fileStream = new MemoryStream())
{
if (destContainer.GetBlockBlobReference(fileName).Exists())
{
destContainer.GetBlockBlobReference(fileName).DownloadToStream(fileStream);
}
var newZip = new ZipArchive(fileStream, ZipArchiveMode.Create);
var entry = newZip.CreateEntry(fileName, CompressionLevel.NoCompression);
using (var innerFile = entry.Open())
{
fileStream.CopyTo(innerFile);
}
fileStream.Close();
}
}
}
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(fixedpartContainer);
CloudBlobContainer destContainer1 = blobClient.GetContainerReference(AzureContainer);
bool isCreated = destContainer1.CreateIfNotExists();
var zipblob = destContainer1.GetBlockBlobReference("inputAssembly.zip");
using (var stream = await zipblob.OpenWriteAsync())
{
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var blobName in blobFileNames)
{
var blob = destContainer.GetBlockBlobClient(blobName);
var zipEntry = zipArchive.CreateEntry(blobName);
using var zipStream = zipEntry.Open();
using var fileStream = new MemoryStream();
await blob.DownloadToAsync(fileStream);
await zipStream.WriteAsync(fileStream.ToArray());
AssemblyCreated = true;
}
}
}

zip all files in azure blob container

I am trying to use azure function to zip all files inside a blob container using System.IO.Compression.
I could list all files inside the container using the below CloudBlob code
CloudStorageAccount storageAccount = CloudStorageAccount.Parse (storageConn);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("<container>");
BlobContinuationToken blobToken = null;
var blobs = await container.ListBlobsSegmentedAsync(blobToken);
var fileList = new List<string>();
var blobpath1 = #"https://<pathtocontainer>/test.zip";
foreach (var blbitem in blobs.Results)
{
if (blbitem is CloudBlockBlob)
{
var blobFileName = blbitem.Uri.Segments.Last().Replace("%20", " ");
var blobFilePath = blbitem.Uri.AbsolutePath.Replace(blbitem.Container.Uri.AbsolutePath + "/", "").Replace("%20", " ");
var blobPath = blobFilePath.Replace("/" + blobFileName, "");
log.LogInformation("blob path : " + blbitem.Uri.ToString());
fileList.Add(blbitem.Uri.ToString());
string rootpath = #"D:\home\site\wwwroot\ZipandSendFile\temp\";
string path = rootpath + blobPath;
log.LogInformation("saving in " + path);
//Add to zip
/*
CloudBlobContainer container = cloudBlobClient.GetContainerReference("<container>");
CloudBlockBlob blob = container.GetBlockBlobReference(blobName);
using (FileStream fs = new FileStream(rootpath, FileMode.Create))
{
blob. DownloadToStream(fs);
}
*/
}
}
</code>
After getting each file details inside the blob , I am trying to add them into zip archive
using the below System.IO.Compression package
My attempt to download file
<code>
public static void AddFilesToZip(string zipPath, string[] files,ILogger log)
{
if (files == null || files.Length == 0)
{
return;
}
log.LogInformation("Executing add files to zip");
log.LogInformation(zipPath);
using (var zipArchive = ZipFile.Open(zipPath, ZipArchiveMode.Update))
{
log.LogInformation("in Zip archive");
foreach (var file in files)
{
var fileInfo = new FileInfo(file);
log.LogInformation(fileInfo.FullName);
zipArchive.CreateEntryFromFile(fileInfo.FullName, fileInfo.Name);
}
}
}
</code>
But I am getting access denied. Any pointers on this ?
Resolved issue by logging into kudo cmdshell and cd into directory and change file attribute
with attrib +A .

Email Attachments missing on Azure, works locally

I have implemented something similar to the following question and I can get it working locally on my server, but when I deploy to Azure it doesn't work. I don't get any errors: just an email without the attachment.
Sending attachments using Azure blobs
Are there restrictions on what types of files can be sent with SendGrid (file is only 56k)?
Does the Azure App service have to be at a particular level or can it be done on Basic?
The blob URL definitley exists and I am setting the stream to zero as suggested in that previous question.
MailMessage mm = new MailMessage("receiver address", "someone");
mm.From = new MailAddress("myAddress", "My Name");
mm.Subject = content.Subject;
mm.Body = content.Body;
mm.IsBodyHtml = true;
mm.BodyEncoding = UTF8Encoding.UTF8;
mm.DeliveryNotificationOptions = DeliveryNotificationOptions.OnFailure;
var attachmentAsStream = _storageAccessService.GetAssetAsStreamForEmail("myContainer", "fileThatExists.pdf");
var attachment = new Attachment(attachmentAsStream, "File.pdf", MediaTypeNames.Application.Pdf);
mm.Attachments.Add(attachment);
public MemoryStream GetAssetAsStreamForEmail(string containerName, string fileName)
{
// Create the blob client.
CloudBlobClient blobClient = StorageAccountReference().CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
var memoryStream = new MemoryStream();
try
{
using (var stream = new MemoryStream())
{
blob.DownloadToStream(memoryStream);
memoryStream.Seek(0, SeekOrigin.Begin);
}
}
catch (Exception ex)
{
Elmah.ErrorSignal.FromCurrentContext().Raise(new Exception("Failed to download Email Atatchment: "+ ex.Message));
}
memoryStream.Position = 0;
return memoryStream;
}
using (SmtpClient client = new SmtpClient())
{
client.Port = 587;
client.Host = "smtp.sendgrid.net";
client.EnableSsl = true;
client.Timeout = 10000;
client.DeliveryMethod = SmtpDeliveryMethod.Network;
client.UseDefaultCredentials = false;
client.Credentials = new System.Net.NetworkCredential("hidden", "hidden");
await client.SendMailAsync(message);
}
Update
Update after Yasir's suggestion below. Downloading the blob from Azure as a Stream only seems to work locally. But if I change to download as a ByteArray then it works everywhere, nonetheless...
public MemoryStream GetAssetAsStreamForEmail(string containerName, string fileName)
{
// Create the blob client.
CloudBlobClient blobClient = StorageAccountReference().CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlockBlob blob = container.GetBlockBlobReference(fileName);
try
{
blob.FetchAttributes();
var fileStream = new byte[blob.Properties.Length];
for (int i = 0; i < blob.Properties.Length; i++)
{
fileStream[i] = 0x20;
}
blob.DownloadToByteArray(fileStream, 0) ;
MemoryStream bufferStream = new MemoryStream(fileStream);
}
catch (Exception ex)
{
Elmah.ErrorSignal.FromCurrentContext().Raise(new Exception("Failed to download Email Atatchment: " + ex.Message));
}
return null;
}
The following code works for me for sending emails using Sendgrid from Azure functions. I attach a CSV file from Blob Storage. It should work for you as well. All you will have to do is ensure that you read the pdf file as a byte[].
public interface IEmailAttachment
{
string Name { get; }
byte[] FileData { get; }
}
public static void Send(MailMessage mailMessage, IEnumerable<IEmailAttachment> attachments)
{
try
{
// Get the configuration data
string server = ConfigReader.EmailServer;
int port = ConfigReader.EmailPort;
string username = ConfigReader.SendGridUserName;
string password = ConfigReader.SendGridPassword;
smtpClient.EnableSsl = false;
smtpClient.Credentials = new NetworkCredential(username, password);
// Create the SMTP Client
SmtpClient smtpClient = new SmtpClient(server, port);
// Prepare the MailMessage
mailMessage.From = new MailAddress(ConfigReader.FromEmail);
var toEmails = ConfigReader.ToEmail.Split(',');
foreach (var toEmail in toEmails)
{
mailMessage.To.Add(toEmail);
}
var ccEmails = ConfigReader.EmailCc.Split(',');
foreach (var ccEmail in ccEmails)
{
mailMessage.CC.Add(ccEmail);
}
// Add attachments
List<MemoryStream> files = new List<MemoryStream>();
if (attachments != null)
{
foreach (IEmailAttachment file in attachments)
{
MemoryStream bufferStream = new MemoryStream(file.FileData);
files.Add(bufferStream);
Attachment attachment = new Attachment(bufferStream, file.Name);
mailMessage.Attachments.Add(attachment);
}
}
mailMessage.IsBodyHtml = true;
// Send the email
smtpClient.Send(mailMessage);
foreach (MemoryStream stream in files)
{
stream.Dispose();
}
}
catch (Exception)
{
throw;
}
}

Resources