I am trying to upload a video from my azure VM to azure blob storage with #azure/storage-blob and using .uploadStream(), but it is not uploading more than 42 bytes
progress { loadedBytes: 42 }
even though the file is 5.7 kb
-rw-r--r-- 1 root root 5.7K Oct 11 10:33 345009243
My upload code is like this
const { BlobServiceClient, StorageSharedKeyCredential } = require("#azure/storage-blob");
const dotenv = require('dotenv');
const getStream = require('into-stream');
//import getStream from 'into-stream'
dotenv.config();
const account = process.env.STORAGE_ACCOUNT;
const accountKey = process.env.ACCOUNT_KEY;
const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey);
const blobServiceClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,sharedKeyCredential
);
const containerName = process.env.CONTAINER_NAME;
const uploadFile = async(filePath,filename) => {
const containerClient = blobServiceClient.getContainerClient(containerName);
const blobName = "vimeoVedio" + new Date().getTime();
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const buffer = Buffer.from(filePath);
const stream = getStream(buffer);
const streamLenght = (1 * 1024 * 1024) / 2;
const uploadBlobResponse = await blockBlobClient.uploadStream(stream,streamLenght, 400, {
// abortSignal: AbortController.timeout(30 * 60 * 1000), // Abort uploading with timeout in 30mins
onProgress: (ev) => {console.log("progress", ev)},
blobHTTPHeaders: {blobContentType: filename.mimeType}
});
return Promise.resolve(`Upload block blob ${blobName} successfully`, uploadBlobResponse.requestId);
}
module.exports = {uploadFile}
I tried this upload with multiple different files with different file sizes but it still is only uploading 42 bytes
I believe the issue is in this line of code:
const buffer = Buffer.from(filePath);
What you are doing here is creating a buffer from a string representing the file path (e.g. C:\temp\myfile.txt) instead of the actual file contents.
You would want to use fs module and read the contents of the file using something like fs.readFile(path[, options], callback) or fs.readFileSync(path\[, options\]). Once you do that, you should be able to upload the file properly.
Related
Is there a way to copy files from Azure Containers (blobs) to Azure File shares?
I was able to copy files from one container to another - see below.
But I wanted to copy files from Blob to File Shares
const {
BlobServiceClient,
StorageSharedKeyCredential
} = require("#azure/storage-blob");
async function copy() {
const account = "<account-name>";
const accountKey = "<account-key>";
const cert = new StorageSharedKeyCredential(account, accountKey)
const blobServiceClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,
cert
);
const sourceContainer = blobServiceClient.getContainerClient("documents")
const desContainer = blobServiceClient.getContainerClient("copy")
//if the desContainer does not exist, please run the following code
// await desContainer.create()
//copy blob
const sourceBlob = sourceContainer.getBlobClient("file1.png");
console.log(sourceBlob, sourceBlob.name)
const desBlob = desContainer.getBlobClient(sourceBlob.name)
const response = await desBlob.beginCopyFromURL(sourceBlob.url);
const result = (await response.pollUntilDone())
console.log(result._response.status)
console.log(result.copyStatus)
}
copy()
I have tested in my environment.
To copy a file from Azure File Share to Azure Blob Storage, you can use the below code:
const {
BlobServiceClient,
StorageSharedKeyCredential,
} = require("#azure/storage-blob");
const {
ShareServiceClient
} = require("#azure/storage-file-share")
async function copy() {
const account = "<account-name>";
const accountKey = "<account-key>";
const cert = new StorageSharedKeyCredential(account, accountKey)
const accountSas = "<account-sas>"
const blobServiceClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,
cert
);
const serviceClient = new ShareServiceClient(`https://${account}.file.core.windows.net${accountSas}`,cert)
const sourceContainer = blobServiceClient.getContainerClient("containerName")
const shareClient = serviceClient.getShareClient("fileShareName")
const directoryClient = shareClient.getDirectoryClient("directoryName");
var fileClient = directoryClient.getFileClient("fileName");
//if the desContainer does not exist, please run the following code
// await desContainer.create()
//copy blob
const sourceBlob = sourceContainer.getBlobClient("blobFileName");
const response = await sourceBlob.beginCopyFromURL(fileClient.url);
}
copy()
To copy the files from Azure Blob Storage to Azure File Share, we can download the blob file to local first and then upload the local file to Azure File Share.
You can use below code to download the blob file to local:
const {
BlobServiceClient,
StorageSharedKeyCredential,
} = require("#azure/storage-blob");
const account = "<account-name>";
const accountKey = "<account-key>";
const cert = new StorageSharedKeyCredential(account, accountKey)
const accountSas = "<account-sas>"
function download() {
const account = "<account-name>";
const accountKey = "<account-key>";
const cert = new StorageSharedKeyCredential(account, accountKey)
const accountSas = "<account-sas>"
const container = "containerName"
const blobFileName = "blobFileName"
const blobServiceClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,
cert
);
const sourceContainer = blobServiceClient.getContainerClient(container)
const sourceBlob = sourceContainer.getBlobClient(blobFileName);
sourceBlob.downloadToFile(blobFileName);
}
download()
You can use the below code to upload the file from local to Azure File Share:
const {
ShareServiceClient
} = require("#azure/storage-file-share");
function upload() {
const account = "<account-name>";
const accountKey = "<account-key>";
const cert = new StorageSharedKeyCredential(account, accountKey)
const accountSas = "<account-sas>"
const serviceClient = new ShareServiceClient(`https://${account}.file.core.windows.net${accountSas}`,cert)
const shareClient = serviceClient.getShareClient("fileShareName")
const directoryClient = shareClient.getDirectoryClient("directoryName");
var fileClient = directoryClient.getFileClient("FileName");
fileClient.uploadFile("localFilePath");
}
upload()
I am trying to upload a file stored in my local system to the Azure blob storage account using Azure Client Library.
Using following code:
const { BlobServiceClient, StorageSharedKeyCredential } = require('#azure/storage-
blob')
const fs = require('fs')
const account = '<account>'
const accountKey = '<SharedKey>'
const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey)
const blobServiceClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,
sharedKeyCredential
)
const containerClient = blobServiceClient.getContainerClient('stream-test-container')
const blockBlobClient = containerClient.getBlockBlobClient('path1/path2/file.xml')
const uploadBlobResponse = blockBlobClient.upload(fs.readFileSync('demo.xml'))
console.log(uploadBlobResponse)
However, I am getting an error that
contentLength cannot be null or undefined.
Can anyone help?
I believe the reason you're getting this error is because you're using incorrect upload method. upload method expects body as HttpRequestBody and contentLength parameters. Since you're not providing the value for contentLength parameter, you're getting this error.
Instead of upload method, you should use uploadData method. It only expects a data buffer that you will get when you read the file. I just tried your code with uploadData method and it worked well for me.
So your code would be:
const { BlobServiceClient, StorageSharedKeyCredential } = require('#azure/storage-blob')
const fs = require('fs')
const account = '<account>'
const accountKey = '<SharedKey>'
const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey)
const blobServiceClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,
sharedKeyCredential
)
const containerClient = blobServiceClient.getContainerClient('stream-test-container')
const blockBlobClient = containerClient.getBlockBlobClient('path1/path2/file.xml')
const uploadBlobResponse = blockBlobClient.uploadData(fs.readFileSync('demo.xml'))
console.log(uploadBlobResponse)
When i try to read files from google bucket and load data in bigquery table, google bucket throws me timeout error. is there a way to read files synchronously and load to bigquery table.
This one works for when files are less, and tried using then as well which also gives same error.
const { BigQuery } = require('#google-cloud/bigquery');
const { Storage } = require('#google-cloud/storage');
var fs = require("fs");
const bucketName = 'bucketname';
const gcpProject = "projectname";
const datasetprojectname = "bqprojectname";
const datasetId = "dsname";
const tableId = "tablename";
exports.helloworld = async (req, res) => {
const bigquery = new BigQuery({ projectId: datasetprojectname });
const storage = new Storage(gcpProject);
const loaddatabq = new Storage(gcpProject);
const bucket = storage.bucket(bucketName);
const fileoptions = {
prefix: "singlefile"
};
var filecount = 0;
var errcount = 0;
var filemoveerrcount = 0;
const [getfilename] = await bucket.getFiles(fileoptions);
var filenamespring = "";
var getjson = null;
getfilename.forEach(async files => {
try {
filecount++;
var filename = files.name;
if (filename != "singlefile/") {
var contents = await files.download(files.name);
await bigquery.dataset(datasetId).table(tableId).insert(JSON.parse(contents).body);
}
}
catch (err) {
}
});
};
If your file are in JSONL (1 JSON document per line, JSON Line), you can use the load job to achieve this.
You can filter on the file that you want by using wildcard character. It will be more efficient than a for loop.
This solution is also cheaper. You are limited to 1500 loads per table and per day, but the load is free. In your current code you use the streaming API, and you pay for it ($0.05 per Gb)
I have created a firebase function that is triggered when a video is uploaded to the firebase storage and by using ffmpeg it add a watermark to it, it works fine with small video sizes but it always timeout in large ones. Any idea how I can overcome these limits
const functions = require('firebase-functions');
const { Storage, Bucket } = require('#google-cloud/storage');
const projectId = 'video-sharing-a57fa';
const admin = require('firebase-admin');
admin.initializeApp();
let gcs = new Storage({
projectId
});
const os = require('os');
const path = require('path');
const spawn = require('child-process-promise').spawn;
exports.addLogo = functions.runWith({ memory: '4GB', timeoutSeconds: 540 }).storage.object().onFinalize(async event => {
const bucket = event.bucket;
const contentType = event.contentType;
const filePath = event.name;
console.log('File change detected, function execution started');
if (path.basename(filePath).startsWith('resized-')) {
console.log('We already renamed that file!');
return;
}
const destBucket = gcs.bucket(bucket);
const tmpFilePath = path.join(os.tmpdir(), path.basename(filePath));
const metadata = { contentType: contentType };
const tmpLogoPath = path.join(os.tmpdir(), 'watermark.png');
await destBucket.file('watermark.png').download({
destination: tmpLogoPath
})
const newPath = path.join(os.tmpdir(), 'output.mp4')
return destBucket.file(filePath).download({
destination: tmpFilePath
}).then(() => {
console.log('entered spawn');
var str = "overlay=10:10"
return spawn('ffmpeg', ['-i', tmpFilePath, '-i', tmpLogoPath, '-filter_complex', str, newPath]);
}).then(() => {
console.log('chaning the name');
return destBucket.upload(newPath, {
destination: path.dirname(filePath) + '/resized-' + path.basename(filePath),
metadata: metadata
})
});
})
Cloud functions have a limited time for execution, it is limited to 9 mins max. More information here. Most likely the problem is that ffmpeg does not manage to add the watermark in time. Your actions should be:
Check the log of the function to confirm that this is exactly the error firebase functions:log --only <FUNCTION_NAME>
Consider different a different architecture option for processing really large files:
a. Limit the amount of data ffmpeg processes, e.g. with -ss 50 -t 10. In this scenario, there will the following architecture: a) one function that read files and put them into a queue, b) one function that reads the size of the file and puts the data into another queue, e.g. {name: "file1.mp4", start: 10, duration: 15}
b. Use an on-demand container such as Cloud Run
c. Use App Engine in case you are constantly processing some files
I am unable to find a way to upload a file not as a stream (buffer, base64) but as a file(png,jgeg,jpg) to Azure Storage Blob.
MY Stream Code is
const blobServiceClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,
sharedKeyCredential, defaultAzureCredentials
);
createBlob = (blobName,blob)=>{
try{
async function main() {
const containerClient = blobServiceClient.getContainerClient('blue');
const content = base64_encode(blob.buffer);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
const uploadBlobResponse = await blockBlobClient.upload(content, content.length);
console.log(`Upload block blob ${blobName} successfully`, uploadBlobResponse.requestId);
return uploadBlobResponse.requestId;
}
main();
}
catch(err){
res.send(err)
}
}
function base64_encode(file) {
// read binary data
//var bitmap = fs.readFileSync(file);
// convert binary data to base64 encoded string
return file.toString('base64');
}
It seems that you were using #azure/storage-blob and your code inspired from Create a blob by uploading data to.
There is a function uploadFile of BlockBlobClient that can help to directly upload a local file to Azure Blob Storage, as the figure below.
Here is my sample code.
const { BlobServiceClient, StorageSharedKeyCredential } = require("#azure/storage-blob");
// Enter your storage account name and shared key
const account = "<your account name>";
const accountKey = "<your account key>";
// Use StorageSharedKeyCredential with storage account and account key
// StorageSharedKeyCredential is only avaiable in Node.js runtime, not in browsers
const sharedKeyCredential = new StorageSharedKeyCredential(account, accountKey);
const blobServiceClient = new BlobServiceClient(
`https://${account}.blob.core.windows.net`,
sharedKeyCredential
);
var containerName = '<your container name>';
var blobName = '<your blob name>';
const containerClient = blobServiceClient.getContainerClient(containerName);
const blockBlobClient = containerClient.getBlockBlobClient(blobName);
var filePath = '<your local file path>';
blockBlobClient.uploadFile(filePath);
You can specify the content type in the options
await blockBlobClient.uploadStream(stream, bufferSize, maxConcurrency, {
blobHTTPHeaders: {
blobContentType: "image/jpeg"
}
})