I have an Azure Topic named "AZTopic" and i have a subscription with three rule description added to it. The rule description has sql filter based on a property priority. When i try to access the data from the subscription i am getting null as broker message.
private static void CreateSubscription(string filterName, string subscriptionName)
{
if (!namespaceManager.SubscriptionExists(Constants.Topic, subscriptionName))
{
namespaceManager.CreateSubscription(Constants.Topic, subscriptionName);
}
SubscriptionClient subscriptionClient = SubscriptionClient.CreateFromConnectionString(connectionString, Constants.Topic, subscriptionName);
var rules = namespaceManager.GetRules(Constants.Topic,Constants.P0Subscription);
foreach (var rule in rules)
{
subscriptionClient.RemoveRule(rule.Name);
}
var P0FilterRule = new RuleDescription()
{
Filter = new SqlFilter(Constants.P0Filter),
Name = "P0"
};
var P1FilterRule = new RuleDescription()
{
Filter = new SqlFilter(Constants.P1Filter),
Name = "P1"
};
var P2FilterRule = new RuleDescription()
{
Filter = new SqlFilter(Constants.P2Filter),
Name = "P2"
};
subscriptionClient.AddRule(P0FilterRule);
subscriptionClient.AddRule(P1FilterRule);
subscriptionClient.AddRule(P2FilterRule);
}
This is my subscription creation code
private QueueMessage GetMessage(string filterName, string subscriptionName)
{
BrokeredMessage receivedMessage = new BrokeredMessage();
QueueMessage messageObject = new QueueMessage();
SubscriptionClient Client = SubscriptionClient.CreateFromConnectionString(Settings.connectionString, Constants.Topic, subscriptionName);
try
{
receivedMessage = Client.Receive();
if (receivedMessage != null)
{
messageObject = receivedMessage.GetBody<QueueMessage>();
receivedMessage.Complete();
}
}
catch (Exception ex)
{
Logger.Log(ex.GetBaseException().Message, LogLevel.Informational, "");
receivedMessage.Abandon();
}
return messageObject;
}
This is my message reading code
Related
I am referring to the Migration Asynchronous Read API that allows creating a read job on SharePoint using CSOM. I am able to create the read job successfully but unfortunately, the job stays in a queue state since long.
The function returns the Object that includes UniqueJobID, AzureContainerManifestUri, AzureQueueReportUri and EncryptionKey
By using clientContext.Site.GetMigrationJobStatus method I am able to check the read job status that always returns Queued
Here is the sample code for reference:
using (var clientContext = new ClientContext(siteUrl))
{
clientContext.Credentials = new SharePointOnlineCredentials(userName, password);
var result = clientContext.Site.CreateSPAsyncReadJob($"{siteUrl}/List/MyList", new AsyncReadOptions { });
clientContext.ExecuteQuery();
MigrationJobState state;
do
{
var status = clientContext.Site.GetMigrationJobStatus(result[0].JobId);
clientContext.ExecuteQuery();
state = status.Value;
} while (state == MigrationJobState.Queued);
}
I have also tried to connect to the AzureQueueReportUri queue that contains the message with encrypted content. I am not sure how we can decrypt the content to make it human readable. Here is the sample message:
{
"Label": "Encrypted",
"JobId": "079ece4a-cfd2-4676-a27d-2662beb5bb0a",
"IV": "RYc+ZA2feX1hnAcVWR1R+w==",
"Content": "qbjTBbb2N+DkNumLoCJSAAfwj8etDLgjxp+b2T9k03L9WfRJKlFBIZO457q+CbHA+8DHJS7VbPzVMoW6ybo2GxgteTYVP+yVUOPPvz57VGQJyzg2gss+Bsjn73GTWWUfwC/W+oWnEpt8PawZysCjSNf6A4HKZKewkskCshN/pND8ZpevrGt2qq0dTt0NkTIkuYv5AvIP7DSWjdl7nN/W5x4c2nR0sPFqKYom41a4tIqrruzwCDEEjWLFtuXAQ+UN2TMV9PWabRFe9n/P1RHrAJaNU+JjJiJm+lE1dQChz+7OuQoJsYnbjYTbqEE8CnIB0/E0zTrc3zLc6th8MBsKpZJjd31ovqr/Xez6zCnvMKotSdScFtTgQqHxmVDBMfMgi2mm8cKQpdKwRufP/YhaDQlvFkmj2FQN0KAMNxwFBh/MWCVhz5uCJ50CGhChcn4h"
}
I am also not able to connect the AzureContainerManifestUri blob container. It fails with an error Authentication Error. Signature did not match.
Can anyone please guide me how can I proceed ahead?
The method parameters have been changed. Here is the latest updated documentation: https://learn.microsoft.com/en-us/sharepoint/dev/apis/export-amr-api
Sample Code: https://gist.github.com/techmadness/484e7de0a7c51e5faf952a79f1eacb85
using System;
using System.Linq;
using System.Security;
using System.Threading;
using Microsoft.SharePoint.Client;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.Queue;
namespace ConsoleApp1
{
internal class Program
{
private static void Main(string[] args)
{
var userName = "admin#tenant.onmicrosoft.com";
var password = GetSecurePassword("password");
var siteUrl = "https://tenant.sharepoint.com/sites/testsite";
var listUrl = $"{siteUrl}/testlist";
var azStorageConnectionStrong = "DefaultEndpointsProtocol=https;AccountName=abcd;AccountKey=xyz";
using (var clientContext = new ClientContext(siteUrl))
{
clientContext.Credentials = new SharePointOnlineCredentials(userName, password);
var azManifestContainer = CreateContainerIfNotExists(azStorageConnectionStrong, "spread-manifest-container");
var azReportQueue = CreateQueueIfNotExists(azStorageConnectionStrong, "spread-report-queue");
var azManifestContainerUrl = GetSASUrl(azManifestContainer);
var azReportQueueUrl = GetSASUrl(azReportQueue);
var output = clientContext.Site.CreateSPAsyncReadJob(
listUrl,
new AsyncReadOptions
{
IncludeDirectDescendantsOnly = true,
IncludeSecurity = true,
},
null,
azManifestContainerUrl,
azReportQueueUrl);
clientContext.ExecuteQuery();
CloudQueueMessage message;
do
{
Thread.Sleep(TimeSpan.FromSeconds(10));
message = azReportQueue.GetMessage();
if (message != null)
{
Console.WriteLine(message.AsString);
azReportQueue.DeleteMessage(message);
}
} while (message != null);
Console.ReadLine();
}
}
private static SecureString GetSecurePassword(string pwd)
{
SecureString securePassword = new SecureString();
foreach (var ch in pwd.ToArray())
{
securePassword.AppendChar(ch);
}
return securePassword;
}
private static CloudBlobContainer CreateContainerIfNotExists(string storageConnectionString, string containerName)
{
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(containerName);
container.CreateIfNotExistsAsync().GetAwaiter().GetResult();
return container;
}
private static CloudQueue CreateQueueIfNotExists(string storageConnectionString, string queueName)
{
var cloudStorageAccount = CloudStorageAccount.Parse(storageConnectionString);
var queueClient = cloudStorageAccount.CreateCloudQueueClient();
var queue = queueClient.GetQueueReference(queueName);
queue.CreateIfNotExistsAsync().GetAwaiter().GetResult();
return queue;
}
public static string GetSASUrl(CloudBlobContainer container)
{
var sharedAccessSignature = container.GetSharedAccessSignature(new SharedAccessBlobPolicy
{
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write,
SharedAccessStartTime = DateTime.UtcNow.AddDays(-1),
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7),
});
return container.StorageUri.PrimaryUri + sharedAccessSignature;
}
public static string GetSASUrl(CloudQueue queue)
{
var sharedAccessSignature = queue.GetSharedAccessSignature(new SharedAccessQueuePolicy
{
Permissions = SharedAccessQueuePermissions.Add | SharedAccessQueuePermissions.Read,
SharedAccessStartTime = DateTime.UtcNow.AddDays(-1),
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(7)
});
return queue.StorageUri.PrimaryUri + sharedAccessSignature;
}
}
}
I have a problem with the Azure Table Storage. What I'm trying to achieve is saving the ChangeToken of the SharePoint list in order to use the webhooks properly.
Here is the code:
public class TablesHelper
{
private static readonly string TokenTableName = "TokenTable";
public static async Task<ListChangeToken> GetChangeTokenForListAsync(string listId)
{
var retrieveOperation = TableOperation.Retrieve<ListChangeToken>("Lists", listId, new List<string>() { "ChangeToken" });
var tableReference = await GetTableReferenceAsync(TokenTableName);
var tableResult = await tableReference.ExecuteAsync(retrieveOperation);
if(tableResult.Result != null)
{
return tableResult.Result as ListChangeToken;
}
return null;
}
public static async Task SaveChangeTokenForListAsync(ListChangeToken changeToken)
{
var insertOperation = TableOperation.Insert(changeToken);
var tableReference = await GetTableReferenceAsync(TokenTableName);
var result = await tableReference.ExecuteAsync(insertOperation);
}
private static async Task<CloudTable> GetTableReferenceAsync(string tableName)
{
var storageAccount = CloudStorageAccount.Parse(ConfigurationHelper.CloudStorage);
var tableClient = storageAccount.CreateCloudTableClient();
var reference = tableClient.GetTableReference(tableName);
await reference.CreateIfNotExistsAsync();
return reference;
}
}
The ListChangeToken class:
public class ListChangeToken : TableEntity
{
public ListChangeToken(string listId, string changeToken)
{
this.PartitionKey = "Lists";
this.RowKey = listId;
this.ChangeToken = changeToken;
}
public ListChangeToken() { }
public string ChangeToken { get; set;}
}
As per request, the function calling TablesHelper:
[FunctionName("EventHandler")]
public static async Task Run([QueueTrigger("events", Connection = "CloudStorage")]string myQueueItem, TraceWriter log)
{
var notificationGroup = Newtonsoft.Json.JsonConvert.DeserializeObject<NotificationGroup>(myQueueItem);
var contextHelper = new ContextHelper();
foreach (var notification in notificationGroup.Value)
{
UriBuilder uriBuilder = new UriBuilder();
uriBuilder.Scheme = "https";
uriBuilder.Host = ConfigurationHelper.TenantDomain;
uriBuilder.Path = notification.SiteUrl;
using (var ctx = contextHelper.GetAppOnlyContext(uriBuilder.ToString()))
{
//Read change token
var currentChangeToken = await TablesHelper.GetChangeTokenForListAsync(notification.Resource);
if(currentChangeToken == null)
{
log.Error($"No change token found for list {notification.Resource}. This is a NO GO. Please use the '/api/Setup' function.");
}
var listId = Guid.Parse(notification.Resource);
var changes = await CSOMHelper.GetListItemChangesAsync(ctx, listId, currentChangeToken.ChangeToken);
if(changes.Count > 0)
{
var lastChange = changes[changes.Count - 1];
//Save the last change token
var changeTokenValue = lastChange.ChangeToken.StringValue;
await TablesHelper.SaveChangeTokenForListAsync(new ListChangeToken(
notification.Resource,
changeTokenValue
));
await HandleChanges(ctx, changes);
}
}
}
log.Info($"C# Queue trigger function processed: {myQueueItem}");
}
The problem is that always, when using the "GetChangeTokenForListAsync" the Entity is received properly, but the .ChangeToken property is always null. It is also not visible when browsing with the Azure Storage Explorer. What am I doing wrong here?
The issue is related to the Azure Storage Emulator (V. 5.7.0.0). The same code works perfectly when working with the "live" Azure.
I am using Azure Function to send out message to multiple Event Hub outputs (from an input EH). My code is the following:
[FunctionName("Gateway")]
public static void Run([EventHubTrigger("%INPUT_HUB_NAME%", Connection = "iothubconnection", ConsumerGroup = "functiontest")]EventData[] eventHubMessage,
[EventHub("%OUT_HUB_NAME%", Connection = "eventhuboutput")]out EventData outputEventHubMessageHotPath,
[EventHub("%OUT_HUB_NAME%", Connection = "eventhuboutput2")]out EventData outputEventHubMessageColdPath,
TraceWriter log)
{
log.Info("**-- Start Azure Func -- **");
foreach (var ehMsg in eventHubMessage)
{
//section to build up the raw section
var rawMessageSection = GetPayload(ehMsg.GetBytes());
var deviceId = GetDeviceId(ehMsg);
log.Info($"Extracted deviceId: {deviceId}");
if (rawMessageSection.aggregates != null)
{
var message = CreateEHMessages("aggregates", rawMessageSection, deviceId, log);
outputEventHubMessageHotPath = message;
outputEventHubMessageColdPath = message;
}
if (rawMessageSection.events != null)
{
outputEventHubMessageColdPath = CreateEHMessages("events", rawMessageSection, deviceId, log);
}
if (rawMessageSection.ipis != null)
{
outputEventHubMessageColdPath = CreateEHMessages("ipis", rawMessageSection, deviceId, log);
}
if (rawMessageSection.errors != null)
{
outputEventHubMessageColdPath = CreateEHMessages("errors", rawMessageSection, deviceId, log);
}
if (rawMessageSection.batteries != null)
{
outputEventHubMessageColdPath = CreateEHMessages("batteries", rawMessageSection, deviceId, log);
}
//await Task.WhenAll(tasks);
}
outputEventHubMessageHotPath = outputEventHubMessageColdPath = null;
}
where:
public static EventData CreateEHMessages(string messageType, dynamic messageBatch, string deviceId, TraceWriter log)
{
var timezone = messageBatch.timezone;
var deviceInstanceId = messageBatch.deviceInstanceId;
int i = 0;
List<dynamic> result = new List<dynamic>();
foreach (var msg in messageBatch[messageType])
{
msg.deviceId = deviceId;
msg.timezone = timezone;
msg.deviceInstanceId = deviceInstanceId;
msg.type = messageType;
result.Add(msg);
i++;
}
var eventData = new EventData(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(result)));
eventData.PartitionKey = deviceInstanceId;
return eventData;
}
The problem is that this Function does NOT seem to publish to EventHub. I have tried to use different syntax of binding but i cannot get it to work.
I suspect is something to do with output bindings but I again tried many options.
Any idea?
You set out parameter multiple times, so all but the last assignments will be lost. But your last assignment is setting them to null, which essentially means you return no messages from your Function.
Have a look at ICollector instead.
Define your output parameters as Collectors:
[EventHub("%OUT_HUB_NAME%", Connection = "eventhuboutput")]
ICollector<EventData> outputEventHubMessageHotPath,
Then add every message to the Collectors, e.g.:
if (rawMessageSection.events != null)
{
outputEventHubMessageHotPath.Add(
CreateEHMessages("events", rawMessageSection, deviceId, log));
}
I'm trying to get Logic Apps metrics like BillableExecutions, Latency etc in my console application.
Currently I'm able to list the logic apps runs, triggers, versions using the .Net Client Microsoft.Azure.Management. But it doesn't seem to have the API to access the monitoring API's.
Code excerpt
private static void Main(string[] args)
{
var token = GetTokenCredentials();
var client = new LogicManagementClient(token, new HttpClientHandler())
{
SubscriptionId = new AzureSubscription().SubscriptionId
};
var dataQuery = new ODataQuery<WorkflowFilter>
{
Top = 50
};
using (client)
{
var logicAppsWorkFlows = client.Workflows.ListBySubscription(dataQuery);
foreach (var logicAppsWorkFlow in logicAppsWorkFlows)
{
var runs = GetWorkflowRuns(client, logicAppsWorkFlow.Id.Split('/')[4], logicAppsWorkFlow.Name);
Console.WriteLine(runs.Count);
}
Console.WriteLine(logicAppsWorkFlows.Count());
}
}
Can someone tell me how to access Logic Apps Metrics? Is there a client similar to Microsoft.Azure.Management for access metrics data?
Update 2
I have found a client dll which was in pre release mode which is used to get metrics. Below is my current code
var token = GetTokenCredentials();
var insightsClient = new InsightsClient(token, new HttpClientHandler())
{
SubscriptionId = new AzureSubscription().SubscriptionId
};
var logicManagementClient = new LogicManagementClient(token, new HttpClientHandler())
{
SubscriptionId = new AzureSubscription().SubscriptionId
};
var dataQuery = new ODataQuery<WorkflowFilter>
{
Top = 50
};
using (logicManagementClient)
{
var logicAppsWorkFlows = logicManagementClient.Workflows.ListBySubscription(dataQuery);
foreach (var logicAppsWorkFlow in logicAppsWorkFlows)
{
using (insightsClient)
{
var metricsDataQuery = new ODataQuery<Metric>
{
Filter = "name.value eq 'ActionLatency' and startTime ge '2014-07-16'"
};
IEnumerable<Metric> metricsList = null;
try
{
metricsList = insightsClient.Metrics.List(logicAppsWorkFlow.Id, metricsDataQuery);
}
catch (Exception e)
{
Console.WriteLine(e);
}
if (metricsList == null) continue;
foreach (var metric in metricsList)
{
foreach (var metricValue in metric.Data)
{
Console.WriteLine(metric.Name.Value + " = " + metricValue.Total);
}
}
}
}
}
I'm getting an exception saying the filter string is not valid. Im referring the filter string structure provided here
https://learn.microsoft.com/en-us/rest/api/monitor/filter-syntax
Can someone tell what im doing wrong here?
Thanks
It looks like ge is not allowed for Logic Apps StartTime field for some reason. I had to change the code to below to make it work
using (logicManagementClient)
{
var logicAppsWorkFlows = logicManagementClient.Workflows.ListBySubscription(dataQuery);
foreach (var logicAppsWorkFlow in logicAppsWorkFlows)
{
using (insightsClient)
{
var metricsDataQuery = new ODataQuery<Metric>
{
Filter = "startTime eq " + DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd") + " and name.value eq 'BillableTriggerExecutions' and endTime eq " + DateTime.Now.ToString("yyyy-MM-dd")
};
var query = metricsDataQuery.GetQueryString();
Console.WriteLine(query);
IEnumerable<Metric> metricsList = null;
try
{
//throws exception if there is no metrics data
//TODO: Check whether the logic app ran atleast one time
metricsList = insightsClient.Metrics.List(logicAppsWorkFlow.Id, metricsDataQuery);
}
catch (Exception e)
{
Console.WriteLine(e);
}
if (metricsList == null) continue;
foreach (var metric in metricsList)
{
foreach (var metricValue in metric.Data)
{
Console.WriteLine(metric.Name.Value + " = " + metricValue.Total);
}
}
}
}
}
I have implemented Azure scheduler by following below blog
http://fabriccontroller.net/a-complete-overview-to-get-started-with-the-windows-azure-scheduler/
Now I am able to create/update/delete jobs in the scheduler but how can I check whether the jobcollection is full? Basically I want to create another collection whenever my current jobcollection is full.
Sharing my code snippet
public class AzureSchedulerStorage : ISchedulerStorage
{
private CertificateCloudCredentials credentials;
//private CloudServiceManagementClient cloudServiceClient;
private string cloudServiceName;
// private IHalseyLogger logger;
public AzureSchedulerStorage(string cloudServiceName, CertificateCloudCredentials credentials)
{
this.cloudServiceName = cloudServiceName;
this.credentials = credentials;
// this.logger = logger;
}
public SchedulerOperationStatusResponse CreateJobCollection(string jobCollectionName)
{
var schedulerServiceClient = new SchedulerManagementClient(credentials);
var jobCollectionCreateParameters = new JobCollectionCreateParameters()
{
Label = jobCollectionName,
IntrinsicSettings = new JobCollectionIntrinsicSettings()
{
Plan = JobCollectionPlan.Standard,
Quota = new JobCollectionQuota()
{
MaxJobCount = 50,
MaxRecurrence = new JobCollectionMaxRecurrence()
{
Frequency = JobCollectionRecurrenceFrequency.Minute
}
}
}
};
var result = schedulerServiceClient.JobCollections.Create(this.cloudServiceName, jobCollectionName, jobCollectionCreateParameters);
return result;
}
public JobCollectionGetResponse GetJobCollection(string jobCollectionName)
{
var schedulerServiceClient = new SchedulerManagementClient(credentials);
var result = schedulerServiceClient.JobCollections.Get(this.cloudServiceName, jobCollectionName);
return result;
}
public void CreateOrUpdate(string jobcollectionName, string jobId, DateTime startDate)
{
var schedulerClient = new SchedulerClient(this.cloudServiceName, jobcollectionName, this.credentials);
var job = new JobCreateOrUpdateParameters()
{
Action = new JobAction()
{
Type = JobActionType.Https,
Request = new JobHttpRequest()
{
Body = "customer=sandrino&command=sendnewsletter",
Headers = new Dictionary<string, string>()
{
{ "Content-Type", "application/x-www-form-urlencoded" },
{ "x-something", "value123" }
},
Method = "POST",
Uri = new Uri("http://postcatcher.in/catchers/527af9acfe325802000001cb"),
}
},
StartTime = startDate,
};
var result = schedulerClient.Jobs.CreateOrUpdate(jobId, job);
}
}
}
After looking at the Scheduler API, it seems that there is no straightforward approach to get the length of a Job Collection.
Probably you can attempt to create a Job and if there is a quota error, then you can create a new Job Collection and add that job.