I am using latest the azure sdk Azure.Storage.Blobs version 12.9.1 for C# . I have to move all blobs from one container to another . I have listed all blobs but I am stuck at how to open an output stream to write a blob in another container .
FileStream fs = File.OpenWrite("\\" + str+"zip");
using (var zipOutputStream = new ZipOutputStream(fs))
{
foreach (var blobFileName in files)
{
zipOutputStream.SetLevel(5);
var blob = dirFromZip.GetBlobClient(blobFileName);
var entry = new ZipEntry(blobFileName);
zipOutputStream.PutNextEntry(entry);
blob.DownloadTo(zipOutputStream);
}
fs.Position = 0;
var blobClient = dirwhereZip.GetBlobClient(str);
blobClient.Upload(fs);
zipOutputStream.Finish();
zipOutputStream.Close();
}
Error : System.NotSupportedException: Stream does not support reading.
Please try the code below:
using System;
using System.Threading.Tasks;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System.IO;
using ICSharpCode.SharpZipLib.Zip;
namespace SO68566758
{
class Program
{
private const string connectionString = "DefaultEndpointsProtocol=https;AccountName=account-name;AccountKey=account-key";
private const string sourceContainer = "source-container";
private const string targetContainer = "target-container";
private const string outputBlobName = "backup.zip";
static async Task Main(string[] args)
{
BlobServiceClient serviceClient = new BlobServiceClient(connectionString);
BlobContainerClient sourceContainerClient = serviceClient.GetBlobContainerClient(sourceContainer);
BlobContainerClient targetContainerClient = serviceClient.GetBlobContainerClient(targetContainer);
var blobs = sourceContainerClient.GetBlobsAsync();
using (var fs = new FileStream(outputBlobName, FileMode.OpenOrCreate))
{
using (var zipOutputStream = new ZipOutputStream(fs))
{
await foreach (var blob in blobs)
{
var blobName = blob.Name;
var blobClient = sourceContainerClient.GetBlobClient(blobName);
//var downloadResponse = await blobClient.DownloadAsync();
//var streamContent = downloadResponse.Value.Content;
var entry = new ZipEntry(blobName);
zipOutputStream.PutNextEntry(entry);
await blobClient.DownloadToAsync(zipOutputStream);
}
}
}
BlobClient targetBlob = targetContainerClient.GetBlobClient(outputBlobName);
using (FileStream fs = new FileStream(outputBlobName, FileMode.Open))
{
await targetBlob.UploadAsync(fs);
}
}
}
}
Related
I am creating a website with ASP.NET Core, and I need to take a bunch of files, zip them together and then upload them to Azure Blobstorage.
It succeeds, I get a downloadlink but when I download the file, I just get a "File" that I cannot open with anything.
Here's the code that handles the uploading:
var moduleVersionReleaseFiles = _appDbContext
.ModuleVersionReleaseFile
.Where(x => x.ModuleVersionReleaseId == moduleVersionReleaseId)
.ToList();
string AzureBlobStorageConfig = _configuration["AzureBlobStorage"];
var connectionString = _configuration["StorageConnectionString"];
string containerName = moduleVersionReleaseId + "-" + Guid.NewGuid().ToString();
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
BlobContainerClient containerClient = await blobServiceClient.CreateBlobContainerAsync(containerName.ToLowerInvariant(), Azure.Storage.Blobs.Models.PublicAccessType.BlobContainer);
BlobClient blobClient = containerClient.GetBlobClient(moduleVersionReleaseId.ToString());
using var stream = new MemoryStream();
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var moduleVersionReleaseFile in moduleVersionReleaseFiles)
{
var entry = archive.CreateEntry(moduleVersionReleaseFile.File);
using (var entryStream = entry.Open())
using (var fileStream = File.OpenRead($"{moduleVersionReleaseFile.Path}\\{moduleVersionReleaseFile.File}"))
{
await fileStream.CopyToAsync(entryStream);
}
}
stream.Position = 0;
await blobClient.UploadAsync(stream, true);
await stream.FlushAsync();
}
return blobClient.Uri.ToString();
I am using Azurite for testing, so I can't log onto the blobstorage and check there.
Why am I not getting a zip file when downloading the file as expected?
The reason you're not able to open the zip file is because it is not getting saved properly. Can you please try with the following code:
using var stream = new MemoryStream();
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var moduleVersionReleaseFile in moduleVersionReleaseFiles)
{
var entry = archive.CreateEntry(moduleVersionReleaseFile.File);
using (var entryStream = entry.Open())
var fileContents = File.ReadAllBytes($"{moduleVersionReleaseFile.Path}\\{moduleVersionReleaseFile.File}"));
using (BinaryWriter zipFileBinary = new BinaryWriter(entryStream))
{
zipFileBinary.Write(fileContents);
}
}
stream.Position = 0;
await blobClient.UploadAsync(stream, true);
}
I have an Azure Storage Account with Azure File share. I want to extract zip archive file to another dir in file share using Azure functions. I wrote this code in C#:
CloudFileDirectory rootDirectory = cloudFileShare.GetRootDirectoryReference();
CloudFileDirectory output = rootDirectory.GetDirectoryReference("output");
CloudFile cloudFile = input.GetFileReference("archive1.zip");
using (var stream = await cloudFile.OpenReadAsync())
{
var file1 = new ZipArchive(stream);
foreach (var zipEntry in file1.Entries)
{
var file2 = output.GetFileReference(zipEntry.Name);
var fileStream = zipEntry.Open();
await file2.UploadFromStreamAsync(fileStream); //error is in this line
}
}
But I got the error:
System.Private.CoreLib: Exception while executing function: HttpTriggerExtract. Microsoft.WindowsAzure.Storage:
Operation is not valid due to the current state of the object.
How to fix this?
Edit: I fix the error using MemoryStream in addition, this code works:
foreach (var zipEntry in file1.Entries) {
var fsz = output.GetFileReference(zipEntry.Name);
using (var ms = new MemoryStream())
{
using (var fileStream = zipEntry.Open())
{
await fileStream.CopyToAsync(ms);
ms.Seek(0, SeekOrigin.Begin);
await fsz.UploadFromStreamAsync(ms);
}
}
Regarding the issue, please refer to the following code (I use package WindowsAzure.Storage 9.3.1 to do that)
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
CloudFileClient cloudFileClient = storageAccount.CreateCloudFileClient();
CloudFileShare cloudFileShare = cloudFileClient.GetShareReference("share2");
CloudFileDirectory rootDirectory = cloudFileShare.GetRootDirectoryReference();
CloudFileDirectory input = rootDirectory.GetDirectoryReference("input");
CloudFileDirectory output = rootDirectory.GetDirectoryReference("output");
CloudFile cloudFile = input.GetFileReference("sample.zip");
using (var stream = await cloudFile.OpenReadAsync())
using (var zipArchive = new ZipArchive(stream)) {
foreach (var entry in zipArchive.Entries)
{
if (entry.Length > 0) {
CloudFile extractedFile = output.GetFileReference(entry.Name);
using (var entryStream = entry.Open())
{
byte[] buffer = new byte[16 * 1024];
using (var ms = await extractedFile.OpenWriteAsync(entry.Length))
{
int read;
while ((read = await entryStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, read);
}
}
}
}
}
}
The above answer helped me for my problem.
With the new Azure Library (12.7.0) you have to code this way:
string srcDir = #"sourcePath";
string destDir = #"sourcePath\testStorageUnzip";
string srcFileName = "AzureStorageZip.zip";
string azureConnectionString = ConfigurationManager.AppSettings["beecloudfileshare_AzureStorageConnectionString"];
StorageSharedKeyCredential credential = BeeFileManager.GetAzureStorageKeyCredential(azureConnectionString);
Uri srcUri = new Uri("https:" + Path.Combine(srcDir, srcFileName).Replace("\\", "/"), UriKind.Absolute);
Uri destDirUri = new Uri("https:" + Path.Combine(destDir).Replace("\\", "/"), UriKind.Absolute);
// Get a reference to the file we created previously
ShareFileClient sourceFile = new ShareFileClient(srcUri, credential);
ShareDirectoryClient shareDirectoryClient = new ShareDirectoryClient(destDirUri,credential);
shareDirectoryClient.CreateIfNotExistsAsync().GetAwaiter().GetResult();
using (var stream = sourceFile.OpenRead())
using (var zipArchive = new ZipArchive(stream))
{
foreach (var entry in zipArchive.Entries)
{
if (entry.Length > 0)
{
//CloudFile extractedFile = output.GetFileReference(entry.Name);
Uri destUri = new Uri("https:" + Path.Combine(destDir, entry.Name).Replace("\\", "/"), UriKind.Absolute);
ShareFileClient extractedFile = new ShareFileClient(destUri, credential);
using (var entryStream = entry.Open())
{
using (MemoryStream ms = new MemoryStream())
{
entryStream.CopyTo(ms);
//
//Sorry I have this part in another method
//
Uri fileUri = new Uri("https:" + Path.GetDirectoryName(filePath).Replace("\\", "/"), UriKind.Absolute);
// Get a reference to the file we created previously
ShareDirectoryClient directory = new ShareDirectoryClient(fileUri, credential);
ShareFileClient file = directory.GetFileClient(Path.GetFileName(filePath));
ms.Seek(0, SeekOrigin.Begin);
file.Create(ms.Length);
file.Upload(ms);
//
//
//
}
}
}
}
}
}
I have multiple files in my Azure storage account say its Master container, I have created a dynamic container which will have the required files copied from Master Container, and those coped files needed to be zipped inside that created container. See below code, where in I have created the zip archive. Also the zip archive is getting created, but when I download manually and see the files, it seems corrupted (for ex. the actual size of the individual files are more than 1Mb but the files which I get after download seems 22Kb), and File formats would be .ipt, .iam (Autodesk Inventor Files)
CloudBlobContainer destContainer = blobClient.GetContainerReference(AzureContainer);
bool isCreated = destContainer.CreateIfNotExists();
var zipblob = destContainer.GetBlockBlobReference("inputAssembly.zip");
using (var stream = await zipblob.OpenWriteAsync())
using (var zip = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var fileName in inputfile)
{
using (var fileStream = new MemoryStream())
{
if (destContainer.GetBlockBlobReference(fileName).Exists())
{
destContainer.GetBlockBlobReference(fileName).DownloadToStream(fileStream);
}
var newZip = new ZipArchive(fileStream, ZipArchiveMode.Create);
var entry = newZip.CreateEntry(fileName, CompressionLevel.NoCompression);
using (var innerFile = entry.Open())
{
fileStream.CopyTo(innerFile);
}
fileStream.Close();
}
}
}
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(fixedpartContainer);
CloudBlobContainer destContainer1 = blobClient.GetContainerReference(AzureContainer);
bool isCreated = destContainer1.CreateIfNotExists();
var zipblob = destContainer1.GetBlockBlobReference("inputAssembly.zip");
using (var stream = await zipblob.OpenWriteAsync())
{
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create))
{
foreach (var blobName in blobFileNames)
{
var blob = destContainer.GetBlockBlobClient(blobName);
var zipEntry = zipArchive.CreateEntry(blobName);
using var zipStream = zipEntry.Open();
using var fileStream = new MemoryStream();
await blob.DownloadToAsync(fileStream);
await zipStream.WriteAsync(fileStream.ToArray());
AssemblyCreated = true;
}
}
}
I am looping through the file names from my database and the same file i have in azure storage. I am zipping those n number of files and download from azure storage. I saves the zipped file to my local storage. When i extract and want to see a file, it say damaged/corrupt.
public ActionResult Download(string productid, string YearActiveid)
{
HomePageModel homepagemodel = new HomePageModel();
homepagemodel.ProdHeaderDetail = GetProductHeaderDetail(productid, YearActiveid);
homepagemodel.PriorYearsActive = GetPriorYearActive(productid, YearActiveid);
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=<name>;AccountKey=<key>;EndpointSuffix=core.windows.net");
CloudBlobClient cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference("product");
var blobFileNames = new string[] { "file1.png", "file2.png", "file3.png", "file4.png" };
var outputMemStream = new MemoryStream();
var zipOutputStream = new ZipOutputStream(outputMemStream);
foreach (var ProdHeaderDetail in homepagemodel.ProdHeaderDetail)
{
zipOutputStream.SetLevel(5);
var blob = cloudBlobContainer.GetBlockBlobReference(ProdHeaderDetail.FileName);
var entry = new ZipEntry(ProdHeaderDetail.FileName);
zipOutputStream.PutNextEntry(entry);
blob.DownloadToStreamAsync(zipOutputStream);
}
zipOutputStream.Finish();
//zipOutputStream.Close();
//zipOutputStream.CloseEntry();
zipOutputStream.IsStreamOwner = false;
outputMemStream.Position = 0;
return File(outputMemStream, "application/zip", "filename.zip");
}
I resolved the issue by adding async and wait
public async Task Download(string productid, string YearActiveid)
await blob.DownloadToStreamAsync(zipOutputStream);
I am referring to the Migration Asynchronous Read API that allows creating a read job on SharePoint using CSOM. I am able to create the read job successfully but unfortunately, the job stays in a queue state since long.
The function returns the Object that includes UniqueJobID, AzureContainerManifestUri, AzureQueueReportUri and EncryptionKey
By using clientContext.Site.GetMigrationJobStatus method I am able to check the read job status that always returns Queued
Here is the sample code for reference:
using (var clientContext = new ClientContext(siteUrl))
{
clientContext.Credentials = new SharePointOnlineCredentials(userName, password);
var result = clientContext.Site.CreateSPAsyncReadJob($"{siteUrl}/List/MyList", new AsyncReadOptions { });
clientContext.ExecuteQuery();
MigrationJobState state;
do
{
var status = clientContext.Site.GetMigrationJobStatus(result[0].JobId);
clientContext.ExecuteQuery();
state = status.Value;
} while (state == MigrationJobState.Queued);
}
I have also tried to connect to the AzureQueueReportUri queue that contains the message with encrypted content. I am not sure how we can decrypt the content to make it human readable. Here is the sample message:
{
"Label": "Encrypted",
"JobId": "079ece4a-cfd2-4676-a27d-2662beb5bb0a",
"IV": "RYc+ZA2feX1hnAcVWR1R+w==",
"Content": "qbjTBbb2N+DkNumLoCJSAAfwj8etDLgjxp+b2T9k03L9WfRJKlFBIZO457q+CbHA+8DHJS7VbPzVMoW6ybo2GxgteTYVP+yVUOPPvz57VGQJyzg2gss+Bsjn73GTWWUfwC/W+oWnEpt8PawZysCjSNf6A4HKZKewkskCshN/pND8ZpevrGt2qq0dTt0NkTIkuYv5AvIP7DSWjdl7nN/W5x4c2nR0sPFqKYom41a4tIqrruzwCDEEjWLFtuXAQ+UN2TMV9PWabRFe9n/P1RHrAJaNU+JjJiJm+lE1dQChz+7OuQoJsYnbjYTbqEE8CnIB0/E0zTrc3zLc6th8MBsKpZJjd31ovqr/Xez6zCnvMKotSdScFtTgQqHxmVDBMfMgi2mm8cKQpdKwRufP/YhaDQlvFkmj2FQN0KAMNxwFBh/MWCVhz5uCJ50CGhChcn4h"
}
I am also not able to connect the AzureContainerManifestUri blob container. It fails with an error Authentication Error. Signature did not match.
Can anyone please guide me how can I proceed ahead?
The method parameters have been changed. Here is the latest updated documentation: https://learn.microsoft.com/en-us/sharepoint/dev/apis/export-amr-api
Sample Code: https://gist.github.com/techmadness/484e7de0a7c51e5faf952a79f1eacb85
using System;
using System.Linq;
using System.Security;
using System.Threading;
using Microsoft.SharePoint.Client;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.Queue;
namespace ConsoleApp1
{
internal class Program
{
private static void Main(string[] args)
{
var userName = "admin#tenant.onmicrosoft.com";
var password = GetSecurePassword("password");
var siteUrl = "https://tenant.sharepoint.com/sites/testsite";
var listUrl = $"{siteUrl}/testlist";
var azStorageConnectionStrong = "DefaultEndpointsProtocol=https;AccountName=abcd;AccountKey=xyz";
using (var clientContext = new ClientContext(siteUrl))
{
clientContext.Credentials = new SharePointOnlineCredentials(userName, password);
var azManifestContainer = CreateContainerIfNotExists(azStorageConnectionStrong, "spread-manifest-container");
var azReportQueue = CreateQueueIfNotExists(azStorageConnectionStrong, "spread-report-queue");
var azManifestContainerUrl = GetSASUrl(azManifestContainer);
var azReportQueueUrl = GetSASUrl(azReportQueue);
var output = clientContext.Site.CreateSPAsyncReadJob(
listUrl,
new AsyncReadOptions
{
IncludeDirectDescendantsOnly = true,
IncludeSecurity = true,
},
null,
azManifestContainerUrl,
azReportQueueUrl);
clientContext.ExecuteQuery();
CloudQueueMessage message;
do
{
Thread.Sleep(TimeSpan.FromSeconds(10));
message = azReportQueue.GetMessage();
if (message != null)
{
Console.WriteLine(message.AsString);
azReportQueue.DeleteMessage(message);
}
} while (message != null);
Console.ReadLine();
}
}
private static SecureString GetSecurePassword(string pwd)
{
SecureString securePassword = new SecureString();
foreach (var ch in pwd.ToArray())
{
securePassword.AppendChar(ch);
}
return securePassword;
}
private static CloudBlobContainer CreateContainerIfNotExists(string storageConnectionString, string containerName)
{
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(containerName);
container.CreateIfNotExistsAsync().GetAwaiter().GetResult();
return container;
}
private static CloudQueue CreateQueueIfNotExists(string storageConnectionString, string queueName)
{
var cloudStorageAccount = CloudStorageAccount.Parse(storageConnectionString);
var queueClient = cloudStorageAccount.CreateCloudQueueClient();
var queue = queueClient.GetQueueReference(queueName);
queue.CreateIfNotExistsAsync().GetAwaiter().GetResult();
return queue;
}
public static string GetSASUrl(CloudBlobContainer container)
{
var sharedAccessSignature = container.GetSharedAccessSignature(new SharedAccessBlobPolicy
{
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write,
SharedAccessStartTime = DateTime.UtcNow.AddDays(-1),
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7),
});
return container.StorageUri.PrimaryUri + sharedAccessSignature;
}
public static string GetSASUrl(CloudQueue queue)
{
var sharedAccessSignature = queue.GetSharedAccessSignature(new SharedAccessQueuePolicy
{
Permissions = SharedAccessQueuePermissions.Add | SharedAccessQueuePermissions.Read,
SharedAccessStartTime = DateTime.UtcNow.AddDays(-1),
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7)
});
return queue.StorageUri.PrimaryUri + sharedAccessSignature;
}
}
}