I am using Azure Function to send out message to multiple Event Hub outputs (from an input EH). My code is the following:
[FunctionName("Gateway")]
public static void Run([EventHubTrigger("%INPUT_HUB_NAME%", Connection = "iothubconnection", ConsumerGroup = "functiontest")]EventData[] eventHubMessage,
[EventHub("%OUT_HUB_NAME%", Connection = "eventhuboutput")]out EventData outputEventHubMessageHotPath,
[EventHub("%OUT_HUB_NAME%", Connection = "eventhuboutput2")]out EventData outputEventHubMessageColdPath,
TraceWriter log)
{
log.Info("**-- Start Azure Func -- **");
foreach (var ehMsg in eventHubMessage)
{
//section to build up the raw section
var rawMessageSection = GetPayload(ehMsg.GetBytes());
var deviceId = GetDeviceId(ehMsg);
log.Info($"Extracted deviceId: {deviceId}");
if (rawMessageSection.aggregates != null)
{
var message = CreateEHMessages("aggregates", rawMessageSection, deviceId, log);
outputEventHubMessageHotPath = message;
outputEventHubMessageColdPath = message;
}
if (rawMessageSection.events != null)
{
outputEventHubMessageColdPath = CreateEHMessages("events", rawMessageSection, deviceId, log);
}
if (rawMessageSection.ipis != null)
{
outputEventHubMessageColdPath = CreateEHMessages("ipis", rawMessageSection, deviceId, log);
}
if (rawMessageSection.errors != null)
{
outputEventHubMessageColdPath = CreateEHMessages("errors", rawMessageSection, deviceId, log);
}
if (rawMessageSection.batteries != null)
{
outputEventHubMessageColdPath = CreateEHMessages("batteries", rawMessageSection, deviceId, log);
}
//await Task.WhenAll(tasks);
}
outputEventHubMessageHotPath = outputEventHubMessageColdPath = null;
}
where:
public static EventData CreateEHMessages(string messageType, dynamic messageBatch, string deviceId, TraceWriter log)
{
var timezone = messageBatch.timezone;
var deviceInstanceId = messageBatch.deviceInstanceId;
int i = 0;
List<dynamic> result = new List<dynamic>();
foreach (var msg in messageBatch[messageType])
{
msg.deviceId = deviceId;
msg.timezone = timezone;
msg.deviceInstanceId = deviceInstanceId;
msg.type = messageType;
result.Add(msg);
i++;
}
var eventData = new EventData(Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(result)));
eventData.PartitionKey = deviceInstanceId;
return eventData;
}
The problem is that this Function does NOT seem to publish to EventHub. I have tried to use different syntax of binding but i cannot get it to work.
I suspect is something to do with output bindings but I again tried many options.
Any idea?
You set out parameter multiple times, so all but the last assignments will be lost. But your last assignment is setting them to null, which essentially means you return no messages from your Function.
Have a look at ICollector instead.
Define your output parameters as Collectors:
[EventHub("%OUT_HUB_NAME%", Connection = "eventhuboutput")]
ICollector<EventData> outputEventHubMessageHotPath,
Then add every message to the Collectors, e.g.:
if (rawMessageSection.events != null)
{
outputEventHubMessageHotPath.Add(
CreateEHMessages("events", rawMessageSection, deviceId, log));
}
Related
In Azure Queue add single entry for queueitem but function called multiple times for same queueitem, not able to understand what is wrong, We have used Async/Await in function and code is also Async, below is Azure function settings,
below is function code,
public class CreateTempVMTempTablesFunction
{
private static Container container;
private static IProcessingFunctionService _processingFunctionService;
private static IAzureFunctionFailuresRepository _azureFunctionFailuresRepository;
private static ITrackLogEventsService _trackLogEventsService;
[FunctionName("CreateTempVMTempTablesFunction")]
public static async Task Run([QueueTrigger("%environment-plan%" + AzureFunctionConstants.CreateTempTableQueue, Connection = "AzureWebJobsStorage")]string myQueueItem, ILogger log)
{
string ErrorMessage = string.Empty;
var start = DateTime.Now;
FunctionStatusEnum IsSuccess = FunctionStatusEnum.Success;
log.LogInformation($"C# Queue trigger function processed: {myQueueItem} - {start}");
Guid tempid = new Guid(myQueueItem);
TempVM tempVM = new TempVM();
try
{
container = BusinessLogic.Helpers.SimpleInjectorWebApiInitializer.InitializeSingleton();;
_processingFunctionService = container.GetInstance<IProcessingFunctionService>();
_azureFunctionFailuresRepository = container.GetInstance<IAzureFunctionFailuresRepository>();
_trackLogEventsService = container.GetInstance<ITrackLogEventsService>();
tempVM = await _processingFunctionService.GetById(tempid);
if (tempVM != null)
{
FunctionStatusEnum IsAlreadyPerformed = await _azureFunctionFailuresRepository.GetAzureFunctionFailureStatus(AzureFunctionConstants.CreateTempVMTempTablesFunction, tempVM.Id);
if (IsAlreadyPerformed != FunctionStatusEnum.Success)
{
ResponseData response = await _processingFunctionService.CreateTempVMTempTables(tempid);
}
else
{
ErrorMessage = AzureFunctionConstants.FunctionAlreadyProcessed;
}
}
else
{
ErrorMessage = AzureFunctionConstants.TempVMNotFound;
}
}
catch (Exception ex)
{
IsSuccess = FunctionStatusEnum.Failed;
ErrorMessage = ex.ToString();
}
finally
{
AzureFunctionFailures azureFunctionFailures = new AzureFunctionFailures()
{
Id = Guid.NewGuid(),
FunctionName = AzureFunctionConstants.CreateTempVMTempTablesFunction,
QueueItem = myQueueItem,
ErrorMessage = ErrorMessage,
StartTime = start,
EndTime = DateTime.Now,
FailureTypeId = tempid,
FunctionStatus = IsSuccess,
ProcessTime = (DateTime.Now - start).TotalMilliseconds,
};
await _azureFunctionFailuresRepository.Add(azureFunctionFailures);
}
log.LogInformation($"End Time : {DateTime.Now} - QueueItem {myQueueItem}");
log.LogInformation($"Total Time : {DateTime.Now - start} - QueueItem {myQueueItem}");
}
}
I have check the code where added entry in queue, but only one entry is added for one queueitem.
This issue happened when added multiple entry in same queue (i.e. load testing where I have added 24 request only) for different queue item, when single queue is run then this is not happened, our functions are in App Service Plan with autoscaling
As mentioned in comments. Just set value of WEBSITE_MAX_DYNAMIC_APPLICATION_SCALE_OUT as 1 in application settings of your function.
I have a problem with the Azure Table Storage. What I'm trying to achieve is saving the ChangeToken of the SharePoint list in order to use the webhooks properly.
Here is the code:
public class TablesHelper
{
private static readonly string TokenTableName = "TokenTable";
public static async Task<ListChangeToken> GetChangeTokenForListAsync(string listId)
{
var retrieveOperation = TableOperation.Retrieve<ListChangeToken>("Lists", listId, new List<string>() { "ChangeToken" });
var tableReference = await GetTableReferenceAsync(TokenTableName);
var tableResult = await tableReference.ExecuteAsync(retrieveOperation);
if(tableResult.Result != null)
{
return tableResult.Result as ListChangeToken;
}
return null;
}
public static async Task SaveChangeTokenForListAsync(ListChangeToken changeToken)
{
var insertOperation = TableOperation.Insert(changeToken);
var tableReference = await GetTableReferenceAsync(TokenTableName);
var result = await tableReference.ExecuteAsync(insertOperation);
}
private static async Task<CloudTable> GetTableReferenceAsync(string tableName)
{
var storageAccount = CloudStorageAccount.Parse(ConfigurationHelper.CloudStorage);
var tableClient = storageAccount.CreateCloudTableClient();
var reference = tableClient.GetTableReference(tableName);
await reference.CreateIfNotExistsAsync();
return reference;
}
}
The ListChangeToken class:
public class ListChangeToken : TableEntity
{
public ListChangeToken(string listId, string changeToken)
{
this.PartitionKey = "Lists";
this.RowKey = listId;
this.ChangeToken = changeToken;
}
public ListChangeToken() { }
public string ChangeToken { get; set;}
}
As per request, the function calling TablesHelper:
[FunctionName("EventHandler")]
public static async Task Run([QueueTrigger("events", Connection = "CloudStorage")]string myQueueItem, TraceWriter log)
{
var notificationGroup = Newtonsoft.Json.JsonConvert.DeserializeObject<NotificationGroup>(myQueueItem);
var contextHelper = new ContextHelper();
foreach (var notification in notificationGroup.Value)
{
UriBuilder uriBuilder = new UriBuilder();
uriBuilder.Scheme = "https";
uriBuilder.Host = ConfigurationHelper.TenantDomain;
uriBuilder.Path = notification.SiteUrl;
using (var ctx = contextHelper.GetAppOnlyContext(uriBuilder.ToString()))
{
//Read change token
var currentChangeToken = await TablesHelper.GetChangeTokenForListAsync(notification.Resource);
if(currentChangeToken == null)
{
log.Error($"No change token found for list {notification.Resource}. This is a NO GO. Please use the '/api/Setup' function.");
}
var listId = Guid.Parse(notification.Resource);
var changes = await CSOMHelper.GetListItemChangesAsync(ctx, listId, currentChangeToken.ChangeToken);
if(changes.Count > 0)
{
var lastChange = changes[changes.Count - 1];
//Save the last change token
var changeTokenValue = lastChange.ChangeToken.StringValue;
await TablesHelper.SaveChangeTokenForListAsync(new ListChangeToken(
notification.Resource,
changeTokenValue
));
await HandleChanges(ctx, changes);
}
}
}
log.Info($"C# Queue trigger function processed: {myQueueItem}");
}
The problem is that always, when using the "GetChangeTokenForListAsync" the Entity is received properly, but the .ChangeToken property is always null. It is also not visible when browsing with the Azure Storage Explorer. What am I doing wrong here?
The issue is related to the Azure Storage Emulator (V. 5.7.0.0). The same code works perfectly when working with the "live" Azure.
I have an Azure Topic named "AZTopic" and i have a subscription with three rule description added to it. The rule description has sql filter based on a property priority. When i try to access the data from the subscription i am getting null as broker message.
private static void CreateSubscription(string filterName, string subscriptionName)
{
if (!namespaceManager.SubscriptionExists(Constants.Topic, subscriptionName))
{
namespaceManager.CreateSubscription(Constants.Topic, subscriptionName);
}
SubscriptionClient subscriptionClient = SubscriptionClient.CreateFromConnectionString(connectionString, Constants.Topic, subscriptionName);
var rules = namespaceManager.GetRules(Constants.Topic,Constants.P0Subscription);
foreach (var rule in rules)
{
subscriptionClient.RemoveRule(rule.Name);
}
var P0FilterRule = new RuleDescription()
{
Filter = new SqlFilter(Constants.P0Filter),
Name = "P0"
};
var P1FilterRule = new RuleDescription()
{
Filter = new SqlFilter(Constants.P1Filter),
Name = "P1"
};
var P2FilterRule = new RuleDescription()
{
Filter = new SqlFilter(Constants.P2Filter),
Name = "P2"
};
subscriptionClient.AddRule(P0FilterRule);
subscriptionClient.AddRule(P1FilterRule);
subscriptionClient.AddRule(P2FilterRule);
}
This is my subscription creation code
private QueueMessage GetMessage(string filterName, string subscriptionName)
{
BrokeredMessage receivedMessage = new BrokeredMessage();
QueueMessage messageObject = new QueueMessage();
SubscriptionClient Client = SubscriptionClient.CreateFromConnectionString(Settings.connectionString, Constants.Topic, subscriptionName);
try
{
receivedMessage = Client.Receive();
if (receivedMessage != null)
{
messageObject = receivedMessage.GetBody<QueueMessage>();
receivedMessage.Complete();
}
}
catch (Exception ex)
{
Logger.Log(ex.GetBaseException().Message, LogLevel.Informational, "");
receivedMessage.Abandon();
}
return messageObject;
}
This is my message reading code
I use sqldependency AND signalR AND thread but it goes on infinite loop
combine BackgroundWorker in global.aspx and sqldependency and signalR
I dont know about problem please help me.
void Application_Start(object sender, EventArgs e)
{
//var session = HttpContext.Current.Session;
//if (session != null && HttpContext.Current != null)
//{
try
{
CSASPNETBackgroundWorker.BackgroundWorker worker = new CSASPNETBackgroundWorker.BackgroundWorker();
worker.DoWork += new CSASPNETBackgroundWorker.BackgroundWorker.DoWorkEventHandler(worker_DoWork);
worker.RunWorker(null);
// This Background Worker is Applicatoin Level,
// so it will keep working and it is shared by all users.
Application["worker"] = worker;
// Code that runs on application startup
}
catch { }
// }
System.Data.SqlClient.SqlDependency.Start(connectionstring);
}
string user_id;
void worker_DoWork(ref int progress,
ref object _result, params object[] arguments)
{
// Do the operation every 1 second wihout the end.
while (true)
{
Random rand = new Random();
int randnum = rand.Next(5000, 20000);
Thread.Sleep(randnum);
// This statement will run every 1 second.
// string user_id = "";
// if (HttpContext.Current != null && HttpContext.Current.Session != null)
{
// user_id = "22";// HttpContext.Current.Session["userId"].ToString();
user_id = ConfigurationManager.AppSettings["session_user_id"].ToString();
updatealarm(user_id);
}
// Other logic which you want it to keep running.
// You can do some scheduled tasks here by checking DateTime.Now.
}
}
AND
public class AlarmInfoRepository {
string connectionstr = ConfigurationManager.ConnectionStrings["officeConnectionString"].ConnectionString;
public IEnumerable GetData(string user_id)
{
using (SqlConnection connection = new SqlConnection(connectionstr))
{
//PersianCalendar jc = new PersianCalendar();
//string todayStr = jc.GetYear(DateTime.Now).ToString("0000") + "/" + jc.GetMonth(DateTime.Now).ToString("00") + "/" +
// jc.GetDayOfMonth(DateTime.Now).ToString("00");
//string timeStr = String.Format("{0:00}:{1:00}:{2:00}", jc.GetHour(DateTime.Now.AddSeconds(10)), jc.GetMinute(DateTime.Now.AddSeconds(10)), jc.GetSecond(DateTime.Now.AddSeconds(10)));
if (connection.State == ConnectionState.Closed)
connection.Open();
using (SqlCommand command = new SqlCommand(#"SELECT dbo.[web_alarmkartable].[id],dbo.[web_alarmkartable].peygir_id,dbo.[web_alarmkartable].user_id,
dbo.[web_alarmkartable].[content],dbo.[web_alarmkartable].[latestcreate_date],dbo.[web_alarmkartable].[latestcreate_time],
dbo.[web_alarmkartable].[firstcreate_date],dbo.[web_alarmkartable].[firstcreate_time],dbo.[web_alarmkartable].[duration],
dbo.[web_alarmkartable].[periodtype_id],dbo.[web_alarmkartable].[period],ISNULL(dbo.[web_alarmkartable].[color],'') AS color,dbo.[web_alarmkartable].id_tel FROM dbo.[web_alarmkartable] where dbo.[web_alarmkartable].content !='' AND dbo.[web_alarmkartable].content IS NOT NULL
AND (dbo.[web_alarmkartable].seen IS NULL OR dbo.[web_alarmkartable].seen =0) AND (dbo.[web_alarmkartable].expier IS NULL OR dbo.[web_alarmkartable].expier =0 )
AND (dbo.[web_alarmkartable].del=0 OR dbo.[web_alarmkartable].del IS NULL) AND dbo.[web_alarmkartable].user_id=" + user_id, connection))
{
// Make sure the command object does not already have
// a notification object associated with it.
command.Notification = null;
SqlDependency dependency = new SqlDependency(command);
dependency.OnChange += new OnChangeEventHandler(dependency_OnChange);
using (var reader = command.ExecuteReader())
return reader.Cast<IDataRecord>()
.Select(x => new AlarmInfo()
{
id = x.GetInt32(0),
peygir_id = x.GetInt32(1),
user_id = x.GetInt32(2),
content = x.GetString(3),
latestcreate_date = x.GetString(4),
latestcreate_time = x.GetString(5),
firstcreate_date = x.GetString(6),
firstcreate_time = x.GetString(7),
duration = x.GetDecimal(8),
periodtype_id = x.GetInt32(9),
period = x.GetDecimal(10),
color = x.GetString(11),
id_tel = x.GetInt32(12)
}).ToList();
}
//if (connection.State == ConnectionState.Open)
// connection.Close();
}
}
private void dependency_OnChange(object sender, SqlNotificationEventArgs e)
{
if ((e.Info.ToString() == "Insert" || e.Info.ToString() == "Update") && e.Type == SqlNotificationType.Change)
AlermHub.Show(e.Info.ToString());
}
}
AND
I'm using the standard example from microsoft for inserting new entitys to a table.
Is there a way to track if a retry was performed?
Code:
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("people");
CustomerEntity customer1 = new CustomerEntity("Harp", "Walter");
TableOperation insertOperation = TableOperation.Insert(customer1);
table.Execute(insertOperation);
With TransientFaultHandlingFramework it was easy to do:
var retryPol = new RetryPolicy<SqlAzureTransientErrorDetectionStrategy>(retryStrategy);
retryPol.Retrying += (obj, eventArgs) =>
{
var msg = String.Format("Retrying, CurrentRetryCount = {0} , Delay = {1}, Exception = {2}", eventArgs.CurrentRetryCount, eventArgs.Delay, eventArgs.LastException.Message);
System.Diagnostics.Debug.WriteLine(msg);
};
You can use the Enterprise Library Transient Fault Handling Application Block as explained in Rob's answer:
var retryPol = new RetryPolicy<StorageTransientErrorDetectionStrategy>(retryStrategy);
retryPol.Retrying += (obj, eventArgs) =>
{
var msg = String.Format("Retrying, CurrentRetryCount = {0} , Delay = {1}, Exception = {2}", eventArgs.CurrentRetryCount, eventArgs.Delay, eventArgs.LastException.Message);
System.Diagnostics.Debug.WriteLine(msg);
};
var options = new TableRequestOptions { RetryPolicy = new RetryPolicies.NoRetry() };
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("people");
CustomerEntity customer1 = new CustomerEntity("Harp", "Walter");
TableOperation insertOperation = TableOperation.Insert(customer1);
retryPol.ExecuteAction<TableResult>(() => { return table.Execute(insertOperation, options); });
If you prefer to use exclusively the Windows Azure Storage Client Library, you can create a custom retry policy that raises events, like this:
public class EventExponentialRetry : IRetryPolicy
{
private static readonly TimeSpan DefaultClientBackoff = TimeSpan.FromSeconds(4.0);
private const int DefaultClientRetryCount = 3;
private TimeSpan deltaBackoff;
private int maximumAttempts;
private ExponentialRetry retry;
public event EventHandler<RetryEventArgs> RaiseRetryEvent;
public EventExponentialRetry()
{
Initialize(DefaultClientBackoff, DefaultClientRetryCount);
}
public EventExponentialRetry(TimeSpan deltaBackoff, int maxAttempts)
{
Initialize(deltaBackoff, maxAttempts);
}
private void Initialize(TimeSpan deltaBackoff, int maxAttempts)
{
this.deltaBackoff = deltaBackoff;
this.maximumAttempts = maxAttempts;
retry = new ExponentialRetry(this.deltaBackoff, this.maximumAttempts);
}
public IRetryPolicy CreateInstance()
{
EventExponentialRetry newInstance = new EventExponentialRetry(this.deltaBackoff, this.maximumAttempts);
newInstance.RaiseRetryEvent = this.RaiseRetryEvent;
return newInstance;
}
public bool ShouldRetry(int currentRetryCount, int statusCode, Exception lastException, out TimeSpan retryInterval, OperationContext operationContext)
{
bool shouldRetry = retry.ShouldRetry(currentRetryCount, statusCode, lastException, out retryInterval, operationContext);
if (shouldRetry)
{
OnRaiseRetryEvent(new RetryEventArgs(currentRetryCount, statusCode, lastException, retryInterval, operationContext));
}
return shouldRetry;
}
protected virtual void OnRaiseRetryEvent(RetryEventArgs e)
{
// Make a temporary copy of the event to avoid possibility of
// a race condition if the last subscriber unsubscribes
// immediately after the null check and before the event is raised.
EventHandler<RetryEventArgs> handler = RaiseRetryEvent;
// Event will be null if there are no subscribers.
if (handler != null)
{
// Use the () operator to raise the event.
handler(this, e);
}
}
}
See the CustomAzureStorageRetryPolicySample GitHub project for a complete sample.
You can make use of the same Enterprise Transient Fault Handling Framework with your Table Storage (and its associated retry notifications).
To do so you disable the default Table Storage transient fault handling and use the enterprise storage fault handling instead.
var retryPol = new RetryPolicy<StorageTransientErrorDetectionStrategy>(retryStrategy);
retryPol.Retrying += (obj, eventArgs) =>
{
var msg = String.Format("Retrying, CurrentRetryCount = {0} , Delay = {1}, Exception = {2}", eventArgs.CurrentRetryCount, eventArgs.Delay, eventArgs.LastException.Message);
System.Diagnostics.Debug.WriteLine(msg);
};
var options = new TableRequestOptions { RetryPolicy = new RetryPolicies.NoRetry() };
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("people");
CustomerEntity customer1 = new CustomerEntity("Harp", "Walter");
TableOperation insertOperation = TableOperation.Insert(customer1);
retryPol.ExecuteAction<TableResult>(() => { return table.Execute(insertOperation, options); });