I am currently working on Internet Of Things, in my current project I was Created the One Azure Cloud Service Project in that I Created the Worker Role, inside the worker role i have wrote below lines of code.
public class WorkerRole : RoleEntryPoint
{
private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
private readonly ManualResetEvent runCompleteEvent = new ManualResetEvent(false);
private static string connectionString;
private static string eventHubName;
public static ServiceClient iotHubServiceClient { get; private set; }
public static EventHubClient eventHubClient { get; private set; }
public override void Run()
{
Trace.TraceInformation("EventsForwarding Run()...\n");
try
{
this.RunAsync(this.cancellationTokenSource.Token).Wait();
}
finally
{
this.runCompleteEvent.Set();
}
}
public override bool OnStart()
{
// Set the maximum number of concurrent connections
ServicePointManager.DefaultConnectionLimit = 12;
// For information on handling configuration changes
// see the MSDN topic at http://go.microsoft.com/fwlink/?LinkId=166357.
bool result = base.OnStart();
Trace.TraceInformation("EventsForwarding OnStart()...\n");
connectionString = ConfigurationManager.AppSettings["Microsoft.ServiceBus.ConnectionString"];
eventHubName = ConfigurationManager.AppSettings["Microsoft.ServiceBus.EventHubName"];
string storageAccountName = ConfigurationManager.AppSettings["AzureStorage.AccountName"];
string storageAccountKey = ConfigurationManager.AppSettings["AzureStorage.Key"];
string storageAccountString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
storageAccountName, storageAccountKey);
string iotHubConnectionString = ConfigurationManager.AppSettings["AzureIoTHub.ConnectionString"];
iotHubServiceClient = ServiceClient.CreateFromConnectionString(iotHubConnectionString);
eventHubClient = EventHubClient.CreateFromConnectionString(connectionString, eventHubName);
var defaultConsumerGroup = eventHubClient.GetDefaultConsumerGroup();
string eventProcessorHostName = "SensorEventProcessor";
EventProcessorHost eventProcessorHost = new EventProcessorHost(eventProcessorHostName, eventHubName, defaultConsumerGroup.GroupName, connectionString, storageAccountString);
eventProcessorHost.RegisterEventProcessorAsync<SensorEventProcessor>().Wait();
Trace.TraceInformation("Receiving events...\n");
return result;
}
public override void OnStop()
{
Trace.TraceInformation("EventsForwarding is OnStop()...");
this.cancellationTokenSource.Cancel();
this.runCompleteEvent.WaitOne();
base.OnStop();
Trace.TraceInformation("EventsForwarding has stopped");
}
private async Task RunAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
//Trace.TraceInformation("EventsToCommmandsService running...\n");
await Task.Delay(1000);
}
}
}
Next I have wrote the below lines of code in SensorEventProcessor, for receiving the messages from event hub and send those messages to IoT hub.
class SensorEventProcessor : IEventProcessor
{
Stopwatch checkpointStopWatch;
PartitionContext partitionContext;
public async Task CloseAsync(PartitionContext context, CloseReason reason)
{
Trace.TraceInformation(string.Format("EventProcessor Shuting Down. Partition '{0}', Reason: '{1}'.", this.partitionContext.Lease.PartitionId, reason.ToString()));
if (reason == CloseReason.Shutdown)
{
await context.CheckpointAsync();
}
}
public Task OpenAsync(PartitionContext context)
{
Trace.TraceInformation(string.Format("Initializing EventProcessor: Partition: '{0}', Offset: '{1}'", context.Lease.PartitionId, context.Lease.Offset));
this.partitionContext = context;
this.checkpointStopWatch = new Stopwatch();
this.checkpointStopWatch.Start();
return Task.FromResult<object>(null);
}
public async Task ProcessEventsAsync(PartitionContext context, IEnumerable<EventData> messages)
{
Trace.TraceInformation("\n");
Trace.TraceInformation("........ProcessEventsAsync........");
//string commandParameterNew = "{\"Name\":\"AlarmThreshold\",\"Parameters\":{\"SensorId\":\"" + "Hello World" + "\"}}";
//await WorkerRole.iotHubServiceClient.SendAsync("astranidevice", new Microsoft.Azure.Devices.Message(Encoding.UTF8.GetBytes(commandParameterNew)));
foreach (EventData eventData in messages)
{
try
{
string jsonString = Encoding.UTF8.GetString(eventData.GetBytes());
Trace.TraceInformation(string.Format("Message received at '{0}'. Partition: '{1}'",
eventData.EnqueuedTimeUtc.ToLocalTime(), this.partitionContext.Lease.PartitionId));
Trace.TraceInformation(string.Format("-->Raw Data: '{0}'", jsonString));
SimpleTemperatureAlertData newSensorEvent = this.DeserializeEventData(jsonString);
Trace.TraceInformation(string.Format("-->Serialized Data: '{0}', '{1}', '{2}', '{3}', '{4}'",
newSensorEvent.Time, newSensorEvent.RoomTemp, newSensorEvent.RoomPressure, newSensorEvent.RoomAlt, newSensorEvent.DeviceId));
// Issuing alarm to device.
string commandParameterNew = "{\"Name\":\"AlarmThreshold\",\"Parameters\":{\"SensorId\":\"" + "Hello World" + "\"}}";
Trace.TraceInformation("Issuing alarm to device: '{0}', from sensor: '{1}'", newSensorEvent.DeviceId, newSensorEvent.RoomTemp);
Trace.TraceInformation("New Command Parameter: '{0}'", commandParameterNew);
await WorkerRole.iotHubServiceClient.SendAsync(newSensorEvent.DeviceId, new Microsoft.Azure.Devices.Message(Encoding.UTF8.GetBytes(commandParameterNew)));
}
catch (Exception ex)
{
Trace.TraceInformation("Error in ProssEventsAsync -- {0}\n", ex.Message);
}
}
await context.CheckpointAsync();
}
private SimpleTemperatureAlertData DeserializeEventData(string eventDataString)
{
return JsonConvert.DeserializeObject<SimpleTemperatureAlertData>(eventDataString);
}
}
When I was debug my code, the ProcessEventsAsync(PartitionContext context, IEnumerable messages) method will never call and just enter into OpenAsync() method then itstop the debugging.
Please tell me Where I did mistake in my project and tell me when the ProcessEventsAsync() method will call.
Regards,
Pradeep
IEventProcessor.ProcessEventsAsync is invoked when there are any unprocessed messages in the EventHub.
An Event Hub contains multiple partitions. A partition is an ordered sequence of events. Within a partition, each event includes an offset. This offset is used by consumers (IEventProcessor) to show the location in the event sequence for a given partition. When an IEventProcessor connects (EventProcessorHost.RegisterEventProcessorAsync), it passes this offset to the Event Hub to specify the location at which to start reading. When there are unprocessed messages (events with higher offset), they are delivered to the IEventProcessor. Checkpointing is used to persist the offset of processed messages (PartitionContext.CheckpointAsync).
You can find detailed information about the internals of EventHub: Azure Event Hubs overview
Have you sent any messages to the EventHub (EventHubClient.SendAsync(EventData))?
Related
Lets say i have a publisher app and it runs and publishes 20 messages and then goes down
import org.apache.pulsar.client.api.Producer;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
public class TestPub {
public static void main(String[] args) throws PulsarClientException, InterruptedException {
PulsarClient client = PulsarClient.builder()
.serviceUrl("pulsar://172.20.170.147:6650")
.build();
Producer<byte[]> producer = client.newProducer()
.topic("example-topic")
.create();
for (int i = 0; i < 20; i++)
{
String msg = ""+i;
producer.newMessage()
.value(msg.getBytes())
.send();
System.out.println("Sent:"+msg);
Thread.sleep(1000);
}
}
}
and then i have a subscriber app which can use either a consumer or reader but it starts after the publisher is done, i need it to read only the latest message published by publisher before i thought this code should do it but somehow it is not working, any clues folks?
import org.apache.pulsar.client.api.*;
public class TestSub {
public static void main(String[] args) throws PulsarClientException {
PulsarClient client = PulsarClient.builder()
.serviceUrl("pulsar://172.20.170.147:6650")
.build();
read(client);
consume(client);
}
static void consume(PulsarClient client) throws PulsarClientException {
Consumer consumer = client.newConsumer()
.topic("example-topic")
.subscriptionName("my-subscription-consumer")
.subscriptionType(SubscriptionType.Exclusive)
.subscriptionInitialPosition(SubscriptionInitialPosition.Latest)
.startMessageIdInclusive()
.subscribe();
//consumer.seek(MessageId.latest);
while (true) {
// Wait for a message
Message msg = consumer.receive();
try {
System.out.println("Message consumed: " +
new String(msg.getData()));
//consumer.acknowledge(msg);
} catch (Exception e) {
consumer.negativeAcknowledge(msg);
}
}
}
static void read(PulsarClient client) throws PulsarClientException {
Reader consumer = client.newReader()
.topic("example-topic")
.subscriptionName("my-subscription-reader")
.startMessageIdInclusive()
.startMessageId(MessageId.latest)
.create()
;
while (true) {
// Wait for a message
Message msg = consumer.readNext();
try {
System.out.println("Message read: " +
new String(msg.getData()));
//consumer.acknowledge(msg);
} catch (Exception e) {
System.out.println(e);
}
}
}
}
Based on the Pulsar API, Message.latest is "MessageId that represents the next message published in the topic." It is NOT the last message published to the topic, but rather the next one to be published. I can see where the naming of this enum could cause that confusion.
So in this scenario the expected behavior is for the readers/consumers to wait until another message is published to the topic before they are triggered to run.
After I send a message to a topic on Azure Service Bus using Spring Integration I would like to get the message id Azure generates. I can do this using JMS. Is there a way to do this using Spring Integration? The code I'm working with:
#Service
public class ServiceBusDemo {
private static final String OUTPUT_CHANNEL = "topic.output";
private static final String TOPIC_NAME = "my_topic";
#Autowired
TopicOutboundGateway messagingGateway;
public String send(String message) {
// How can I get the Azure message id after sending here?
this.messagingGateway.send(message);
return message;
}
#Bean
#ServiceActivator(inputChannel = OUTPUT_CHANNEL)
public MessageHandler topicMessageSender(ServiceBusTopicOperation topicOperation) {
DefaultMessageHandler handler = new DefaultMessageHandler(TOPIC_NAME, topicOperation);
handler.setSendCallback(new ListenableFutureCallback<>() {
#Override
public void onSuccess(Void result) {
System.out.println("Message was sent successfully to service bus.");
}
#Override
public void onFailure(Throwable ex) {
System.out.println("There was an error sending the message to service bus.");
}
});
return handler;
}
#MessagingGateway(defaultRequestChannel = OUTPUT_CHANNEL)
public interface TopicOutboundGateway {
void send(String text);
}
}
You could use ChannelInterceptor to get message headers:
public class CustomChannelInterceptor implements ChannelInterceptor {
#Override
public Message<?> preSend(Message<?> message, MessageChannel channel) {
//key of the message-id header is not stable, you should add logic here to check which header key should be used here.
//ref: https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/spring/azure-spring-cloud-starter-servicebus#support-for-service-bus-message-headers-and-properties
String messageId = message.getHeaders().get("message-id-header-key").toString();
return ChannelInterceptor.super.preSend(message, channel);
}
}
Then in the configuration, set this interceptor to your channel
#Bean(name = OUTPUT_CHANNEL)
public BroadcastCapableChannel pubSubChannel() {
PublishSubscribeChannel channel = new PublishSubscribeChannel();
channel.setInterceptors(Arrays.asList(new CustomChannelInterceptor()));
return channel;
}
I have a simple Azure Function which returns to a queue:
private readonly TelemetryClient _telemetryClient;
[return: Queue("%ReturnQueue%")]
public async Task<string> Run([QueueTrigger("%RequestQueue%")] string msg, ILogger log)
{
try
{
//Some dependency calls
}
catch(Exception ex)
{
var dic = new Dictionary<string,string>();
dic.Add("Id", someId);
dic.Add("CustomData", cusomData);
_telemetryClient.TrackException(ex, dic);
}
}
I obviously get a compilation error saying that not all code paths returns a value.
The problem is that if I add a throw at the end of the catch block the Azure Functions runtime replicate the excpetion on the appinsights portal. How can I add custom data to my exceptions like this?
You can create your own Exception type:
public class MyCustomException : Exception
{
public string Id {get;set;}
public string CustomData {get;set;}
public Exception RootException {get;set;}
public MyCustomException(string id, string customData, Exception ex)
{
Id = id;
CustomData = customData;
RootException = ex;
}
}
private readonly TelemetryClient _telemetryClient;
[return: Queue("%ReturnQueue%")]
public async Task<string> Run([QueueTrigger("%RequestQueue%")] string msg, ILogger log)
{
try
{
//Some dependency calls
}
catch(Exception ex)
{
//var dic = new Dictionary<string,string>();
//dic.Add("Id", someId);
//dic.Add("CustomData", cusomData);
var customEx = new MyCustomException(someId, cusomData, ex);
_telemetryClient.TrackException(customEx);
}
finally
{
return "";
}
}
PS: inside MyCustomException you can actually have Dictionary rather than string properties.
I am having an existing webjob(V3.0) in .net core that has a function that is invoked by manual trigger, essentially by a webhook. I want to add another function to the same webjob that should be invoked on a Timer trigger every 20 mins. Is it possible to have both these in the same webjob. If it is possible what should the host configuration that I need to do. I tried going through Microsoft's documentation but there is barely any documentation with respect to the host configuration part with multiple triggers
Yes but your function would need to be triggered by something in Azure Storage like a Queue. This code is probably more then you might need. All of my services implement a custom interface IServiceInvoker. My CTOR asks for an
IEnumerable<IServiceInvoker>
which gets all of the services. Then I either use a constant or a passed in value to determine what service to run. Since I ONLY want ONE function to ever be running I am using the Singleton attribute passing in String.Empty. I also have the following settings on my Queues
b.AddAzureStorage(a =>
{
a.BatchSize = 1;
a.NewBatchThreshold = 1;
a.MaxDequeueCount = 1;
a.MaxPollingInterval = TimeSpan.FromSeconds(60);
a.VisibilityTimeout = TimeSpan.FromSeconds(60);
});
Finally I found that during testing I sometimes needed to turn off on or more functions hence the class ServiceConfigurationProvider.
Code sample follows I removed quite a bit of code so YMMV
public static async Task Main(string[] args)
{
await CreateHostBuilder(args).Build().RunAsync();
}
more code
public class Functions
{
/// <summary>
/// This scopes the singleton attribute to each individual function rather than the entire host
/// </summary>
const String SCOPESINGLETONTOFUNCTION = "";
readonly ILogger<Functions> _logger;
readonly Dictionary<String, IServiceInvoker> _services;
readonly IConfiguration _configuration;
private Functions()
{
_services = new Dictionary<string, IServiceInvoker>();
}
public Functions(IEnumerable<IServiceInvoker> services, ILoggerFactory loggerFactory, IServiceProvider serviceProvider, IConfiguration configuration) : this()
{
_logger = loggerFactory.CreateLogger<Functions>();
foreach (var service in services)
{
_services.Add(service.ServiceIdentifier, service);
}
_configuration = configuration;
}
[Disable(typeof(ServiceConfigurationProvider))]
[Singleton(SCOPESINGLETONTOFUNCTION)]
public async Task TimerTriggerFunction([TimerTrigger("%TimerTriggerFunctionExpression%")]TimerInfo myTimer, CancellationToken cancellationToken)
{
try
{
if (_services.TryGetValue("ServiceName", out IServiceInvoker serviceToInvoke))
{
await serviceToInvoke.InvokeServiceAsync(null, cancellationToken, false);
}
}
catch (Exception ex)
{
_logger?.LogError(ex, $"Unhandled exception occurred in method:'{nameof(TimerTriggerFunction)}'");
}
}
[Disable(typeof(ServiceConfigurationProvider))]
[Singleton(SCOPESINGLETONTOFUNCTION)]
public async Task ServiceInvokerQueueFunction([QueueTrigger("%ServiceInvokerQueueName%", Connection = "AzureWebJobsStorage")] ServiceInvokerMessage serviceInvokerMessage, CancellationToken cancellationToken)
{
if (serviceInvokerMessage is null || String.IsNullOrEmpty(serviceInvokerMessage.ServiceIdentifier))
{
_logger?.LogError("The queue message received in the ServiceInvokerQueueFunction could not be serialized into a ServiceInvokerMessage instance.");
}
else
{
Boolean serviceExists = _services.TryGetValue(serviceInvokerMessage.ServiceIdentifier, out IServiceInvoker serviceToInvoke);
if (serviceExists)
{
try
{
await serviceToInvoke.InvokeServiceAsync(null, cancellationToken, true);
}
catch (Exception exception)
{
_logger?.LogError(exception, $"Unhandled exception occurred in method:'{nameof(ServiceInvokerQueueFunction)}' for service:'{serviceInvokerMessage.ServiceIdentifier}'");
}
}
}
}
[Disable(typeof(ServiceConfigurationProvider))]
[Singleton(SCOPESINGLETONTOFUNCTION)]
public async Task RecordQueueFunction([QueueTrigger("%RecordQueueName%", Connection = "RecordConnectString")] string message, CancellationToken cancellationToken)
{
{
_logger?.LogInformation(message);
try
{
if (_services.TryGetValue("ServiceName", out IServiceInvoker serviceToInvoke))
{
await serviceToInvoke.InvokeServiceAsync(message, cancellationToken, false);
}
}
catch (Exception ex)
{
_logger?.LogError(ex, $"Unhandled exception occurred in method:'{nameof(RecordQueueFunction)}'");
throw;
}
}
}
}
public class ServiceConfigurationProvider
{
readonly IConfiguration _configuration;
public ServiceConfigurationProvider(IConfiguration configuration)
{
_configuration = configuration;
}
public bool IsDisabled(MethodInfo method)
{
Boolean returnValue = false;
String resultConfiguration = _configuration[$"{method.Name}Disable"];
if (!String.IsNullOrEmpty(resultConfiguration))
{
Boolean.TryParse(resultConfiguration, out returnValue);
}
return returnValue;
}
}
Do You have any idea how to log all outgoing/incoming messages? I am not sure how to capture outgoing messages.
I use Chains and Forms.
For example
await Conversation.SendAsync(activity, rootDialog.BuildChain);
AND
activity.CreateReply(.....);
I found better solution
public class BotToUserLogger : IBotToUser
{
private readonly IMessageActivity _toBot;
private readonly IConnectorClient _client;
public BotToUserLogger(IMessageActivity toBot, IConnectorClient client)
{
SetField.NotNull(out _toBot, nameof(toBot), toBot);
SetField.NotNull(out _client, nameof(client), client);
}
public IMessageActivity MakeMessage()
{
var toBotActivity = (Activity)_toBot;
return toBotActivity.CreateReply();
}
public async Task PostAsync(IMessageActivity message, CancellationToken cancellationToken = default(CancellationToken))
{
await _client.Conversations.ReplyToActivityAsync((Activity)message, cancellationToken);
}
}
public class BotToUserDatabaseWriter : IBotToUser
{
private readonly IBotToUser _inner;
public BotToUserDatabaseWriter(IBotToUser inner)
{
SetField.NotNull(out _inner, nameof(inner), inner);
}
public IMessageActivity MakeMessage()
{
return _inner.MakeMessage();
}
public async Task PostAsync(IMessageActivity message, CancellationToken cancellationToken = default(CancellationToken))
{
// loging outgoing message
Debug.WriteLine(message.Text);
//TODO log message for example into DB
await _inner.PostAsync(message, cancellationToken);
}
In controller use
public MessagesController()
{
var builder = new ContainerBuilder();
builder.RegisterType<BotToUserLogger>()
.AsSelf()
.InstancePerLifetimeScope();
builder.Register(c => new BotToUserTextWriter(c.Resolve<BotToUserLogger>()))
.AsImplementedInterfaces()
.InstancePerLifetimeScope();
builder.Update(Microsoft.Bot.Builder.Dialogs.Conversation.Container);
}
Its look like I cant log outgoing message.
I changed SDK source code.
Add event in Conversations.cs
For example like this.
public delegate void MessageSendedEventHandler(object sender, Activity activity, string conversationId);
public static event MessageSendedEventHandler MessageSended;
And add in every Send....HttpMessagesAsync method
this MessageSended?.Invoke(this, activity, conversationId);
its not great solution. But its working