I am using entlib 5.0.1. I have created a cache using code configuration
IContainerConfigurator configurator =
new UnityContainerConfigurator(_unityContainer);
configurator.ConfigureCache(item.PartitionName, item.MaxItemNumber);
CacheBlockImp block = new CacheBlockImp(
_unityContainer.Resolve<ICacheManager>(item.PartitionName),
item.PartitionType);
I saw a strange behavior with the size limit, I have configured the cache to hold 15 items, I am adding in a loop 18 items, I would suspect afer the addition to have only 15 items, but I get 8, when I have added a refresh action - to be notify when 1 item was evacuated, I really see that 7 of them were evacuated, all the items have the same priority.
class Program
{
static void Main(string[] args)
{
var unityContainer = new UnityContainer();
IContainerConfigurator configurator = new UnityContainerConfigurator(unityContainer);
configurator.ConfigureCache("Test", 10);
var cache = unityContainer.Resolve<ICacheManager>("Test");
for (int i = 0; i < 18; i++)
{
var dummy = new Dummy()
{
ID = i,
Data = "hello " + i.ToString()
};
cache.Add(i.ToString(), dummy, CacheItemPriority.Normal, null, null);
}
Thread.Sleep(1000);
int count = cache.CachedItems().Count;
}
public class Dummy
{
public int ID { get; set; }
public string Data { get; set; }
}
}
public static class CacheBlockExtension
{
public static void ConfigureCache(this IContainerConfigurator configurator, string configKey,
int maxNumOfItems)
{
ConfigurationSourceBuilder builder = new ConfigurationSourceBuilder();
DictionaryConfigurationSource configSource = new DictionaryConfigurationSource();
// simple inmemory cache configuration
builder.ConfigureCaching().ForCacheManagerNamed(configKey).WithOptions
.StartScavengingAfterItemCount(maxNumOfItems)
.StoreInMemory();
builder.UpdateConfigurationWithReplace(configSource);
EnterpriseLibraryContainer.ConfigureContainer(configurator, configSource);
}
public static List<object> CachedItems(this ICacheManager cachemanger)
{
Microsoft.Practices.EnterpriseLibrary.Caching.Cache cache =
(Microsoft.Practices.EnterpriseLibrary.Caching.Cache)cachemanger.GetType().GetField("realCache", System.Reflection.BindingFlags.Instance |
System.Reflection.BindingFlags.NonPublic).GetValue(cachemanger);
List<object> tmpret = new List<object>();
foreach (DictionaryEntry Item in cache.CurrentCacheState)
{
Object key = Item.Key;
CacheItem cacheItem = (CacheItem)Item.Value;
tmpret.Add(cacheItem.Value);
}
return tmpret;
}
Related
I found this post describing how to conditionally copy values over a destination object if they are not null.
Works great except for list members, it always overwrites them with an empty list. I'm not sure if I've just not configured the mapper correctly or if this is a bug. The following program demonstrates the issue.
namespace automapper_test
{
using AutoMapper;
using System;
using System.Collections.Generic;
class Program
{
class Test
{
public int? A { get; set; }
public string B { get; set; }
public Guid? C { get; set; }
public List<Guid> D { get; set; }
}
static void Main(string[] args)
{
var config = new MapperConfiguration(cfg =>
{
cfg.AllowNullCollections = true;
cfg.CreateMap<Test, Test>().ForAllMembers(opt => opt.Condition((src, dest, member) => member != null));
});
var mapper = config.CreateMapper();
var source = new Test { A = 2, C = Guid.Empty };
var target = new Test { A = 1, B = "hello", C = Guid.NewGuid(), D = new List<Guid> { Guid.NewGuid() } };
mapper.Map(source, target);
System.Diagnostics.Debug.Assert(target.D.Count == 1);
}
}
}
I'm trying to find out the reason of problems of my .net core application with protobuf serialization, so i've wrote a test application and... there is some strange. There is a memory leak if i use Task and there is no memory leak if i use Thread.
Example with Task:
internal class Program
{
private static void Main(string[] args)
{
var data = new Person
{
Id = 1,
Name = new string('q', 10000),
Address = new Address
{
Line1 = new string('w', 10000),
Line2 = new string('e', 10000)
}
};
//for (var i = 0; i < 100; i++)
//{
// var thread = new Thread(() =>
// {
// while (true)
// try
// {
// var b = ProtoSerialize(data);
// Person serializedPerson;
// using (Stream stream2 = new MemoryStream(b))
// {
// serializedPerson = Serializer.Deserialize<Person>(stream2);
// }
// }
// catch (Exception e)
// {
// Console.WriteLine(e);
// }
// });
// thread.Start();
//}
for (var i = 0; i < 100; i++)
{
new Task(() =>
{
while (true)
{
var b = ProtoSerialize(data);
Person serializedPerson;
using (Stream stream2 = new MemoryStream(b))
{
serializedPerson = Serializer.Deserialize<Person>(stream2);
}
}
}).Start();
}
Console.ReadLine();
}
public static byte[] ProtoSerialize<T>(T record) where T : class
{
using (var stream = new MemoryStream())
{
Serializer.Serialize(stream, record);
return stream.ToArray();
}
}
}
[ProtoContract]
public class Person
{
[ProtoMember(1)]
public int Id { get; set; }
[ProtoMember(2)]
public string Name { get; set; }
[ProtoMember(3)]
public Address Address { get; set; }
}
[ProtoContract]
public class Address
{
[ProtoMember(1)]
public string Line1 { get; set; }
[ProtoMember(2)]
public string Line2 { get; set; }
}
and screenshot of dotMemory:
so if i use threads (commented code)
for (var i = 0; i < 100; i++)
{
var thread = new Thread(() =>
{
while (true)
try
{
var b = ProtoSerialize(data);
Person serializedPerson;
using (Stream stream2 = new MemoryStream(b))
{
serializedPerson = Serializer.Deserialize<Person>(stream2);
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
});
thread.Start();
}
Then there is no leak:
So the question is:
Am i doing something wrong?
Why this happens?
How to do wright?
p.s.
protobuf-net is 2.3.2
.net core 2.0
UPD1: call GC.Collect in main thread changes nothing (there is still memory leak)
Well that's fun. protobuf-net does make limited use of some background pools - there is a [ThreadStatic] instance of ProtoReader, for example - although that is a light object by itself; while it does keep a byte[] pool handy, that is not kept referenced by the [ThreadStatic] instance (the buffer is released to the pool before then). So there's nothing that immediately leaps to mind; have you tried doing a few forced GC.Collects to see if anything is actually leaking, vs simply not been collected yet? Have you tried replacing all the protobuf-net with dummy code (Thread.Sleep(100); for example) to see if you're just seeing the weight of the TPL machinery? Do you know (from that tool) what object types are accounting for the overhead?
It is also possible that this is simply a consequence of the very different threading usage here upsetting GC: the Task code is going to be using the ThreadPool, so a limited number of threads. That is neither good nor bad - just: different.
I'm trying to get at the individual 32bit samples using CScore. What I have so far is
public MainWindow()
{
InitializeComponent();
var wasapiCapture = new WasapiCapture();
wasapiCapture.Initialize();
wasapiCapture.Start();
var wasapiCaptureSource = new SoundInSource(wasapiCapture);
var stereoSource = wasapiCaptureSource.ToStereo();
var ieeeFloatToSample = new IeeeFloatToSample(stereoSource);
var sampleProvider = new SampleProvider(ieeeFloatToSample);
var wavesource = sampleProvider.ToWaveSource();
var wasapiOut = new WasapiOut();
wasapiOut.Initialize(wavesource);
wasapiOut.Play();
}
And a class
class SampleProvider : ISampleSource
{
private ISampleSource _source;
public SampleProvider(ISampleSource source)
{
this._source = source;
}
public int Read(float[] buffer, int offset, int count)
{
var sampleRead = _source.Read(buffer, 0, count);
return sampleRead;
}
public void Dispose()
{
throw new NotImplementedException();
}
public WaveFormat WaveFormat { get; private set; }
public long Position { get; set; }
public long Length { get; private set; }
}
Which I thought would pass the audio through unchanged but I am getting an error on the sampleProvider.ToWaveSource(); saying "Object reference not set to an instance of an object"
Any ideas? Thanks.
If you poke around through the CSCore source you'll find that ToWaveSource ends up cloning the WaveFormat of your SampleProvider - which you've left undefined. You can probably just return the WaveFormat from the upstream source:
public WaveFormat WaveFormat { get { return _source.WaveFormat; } }
Are there any chances to have the MultiRecordEngine set the line number for each record?
I read the documentation and found that the Engine has a LineNumber property, but I couldn't find a way to use it.
What I would like to have:
i.e.:
[FixedLengthRecord]
public class Employee
{
public int LineNumber; // <-- setup by the engine while reading the file
[FieldFixedLength(1)]
public string Type = "2";
[FieldFixedLength(6)]
[FieldTrim(TrimMode.Left, "0")]
public int ID;
}
Or... can I rely on the index of the record in the collection created by the Engine.ReadFile?
i.e.:
static void Main(string[] args)
{
var engine = new MultiRecordEngine(CustomSelector, _types);
var obj = engine.ReadFile("order.txt");
// obj.GetValue(100) returns same record found on the 101th line in the file?
}
You can use the AfterReadRecord event to set the LineNumber property.
Here is a working example
public class Employee
{
[FieldIgnored]
public int LineNumber; // <-- setup by the engine while reading the file
[FieldFixedLength(1)]
public string Type = "2";
[FieldFixedLength(6)]
[FieldTrim(TrimMode.Left, "0")]
public int ID;
}
class Program
{
static void Main(string[] args)
{
var engine = new FileHelperEngine<Employee>();
engine.AfterReadRecord += engine_AfterReadRecord;
Employee[] records = engine.ReadString("2000003" + Environment.NewLine
+ "5000006");
Employee firstRecord = records[0];
Assert.AreEqual(1, firstRecord.LineNumber);
Assert.AreEqual("2", records[0].Type);
Assert.AreEqual(3, records[0].ID);
Employee secondRecord = records[1];
Assert.AreEqual(2, secondRecord.LineNumber);
Assert.AreEqual("5", records[1].Type);
Assert.AreEqual(6, records[1].ID);
Console.Read();
}
static void engine_AfterReadRecord(EngineBase engine, AfterReadEventArgs<Employee> e)
{
e.Record.LineNumber = engine.LineNumber;
}
}
We notice that inside of our .Net application we have contention when it comes to using SqlDataReader. While we understand that SqlDataReader is not ThreadSafe, it should scale. The following code is a simple example to show that we cannot scale our application because there is contention on the SqlDataReader GetValue method. We are not bound by CPU, Disk, or Network; Just the internal contention on the SqlDataReader. We can run the application 10 times with 1 thread and it scales linearly, but 10 threads in 1 app does not scale. Any thoughts on how to scale reading from SQL Server in a single c# application?
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Diagnostics;
using System.Globalization;
namespace ThreadAndSQLTester
{
class Host
{
/// <summary>
/// Gets or sets the receive workers.
/// </summary>
/// <value>The receive workers.</value>
internal List<Worker> Workers { get; set; }
/// <summary>
/// Gets or sets the receive threads.
/// </summary>
/// <value>The receive threads.</value>
internal List<Thread> Threads { get; set; }
public int NumberOfThreads { get; set; }
public int Sleep { get; set; }
public int MinutesToRun { get; set; }
public bool IsRunning { get; set; }
private System.Timers.Timer runTime;
private object lockVar = new object();
public Host()
{
Init(1, 0, 0);
}
public Host(int numberOfThreads, int sleep, int minutesToRun)
{
Init(numberOfThreads, sleep, minutesToRun);
}
private void Init(int numberOfThreads, int sleep, int minutesToRun)
{
this.Workers = new List<Worker>();
this.Threads = new List<Thread>();
this.NumberOfThreads = numberOfThreads;
this.Sleep = sleep;
this.MinutesToRun = minutesToRun;
SetUpTimer();
}
private void SetUpTimer()
{
if (this.MinutesToRun > 0)
{
this.runTime = new System.Timers.Timer();
this.runTime.Interval = TimeSpan.FromMinutes(this.MinutesToRun).TotalMilliseconds;
this.runTime.Elapsed += new System.Timers.ElapsedEventHandler(runTime_Elapsed);
this.runTime.Start();
}
}
void runTime_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
this.runTime.Stop();
this.Stop();
this.IsRunning = false;
}
public void Start()
{
this.IsRunning = true;
Random r = new Random(DateTime.Now.Millisecond);
for (int i = 0; i < this.NumberOfThreads; i++)
{
string threadPoolId = Math.Ceiling(r.NextDouble() * 10).ToString();
Worker worker = new Worker("-" + threadPoolId); //i.ToString());
worker.Sleep = this.Sleep;
this.Workers.Add(worker);
Thread thread = new Thread(worker.Work);
worker.Name = string.Format("WorkerThread-{0}", i);
thread.Name = worker.Name;
this.Threads.Add(thread);
thread.Start();
Debug.WriteLine(string.Format(CultureInfo.InvariantCulture, "Started new Worker Thread. Total active: {0}", i + 1));
}
}
public void Stop()
{
if (this.Workers != null)
{
lock (lockVar)
{
for (int i = 0; i < this.Workers.Count; i++)
{
//Thread thread = this.Threads[i];
//thread.Interrupt();
this.Workers[i].IsEnabled = false;
}
for (int i = this.Workers.Count - 1; i >= 0; i--)
{
Worker worker = this.Workers[i];
while (worker.IsRunning)
{
Thread.Sleep(32);
}
}
foreach (Thread thread in this.Threads)
{
thread.Abort();
}
this.Workers.Clear();
this.Threads.Clear();
}
}
}
}
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data.SqlClient;
using System.Data;
using System.Threading;
using System.ComponentModel;
using System.Data.OleDb;
namespace ThreadAndSQLTester
{
class Worker
{
public bool IsEnabled { get; set; }
public bool IsRunning { get; set; }
public string Name { get; set; }
public int Sleep { get; set; }
private string dataCnString { get; set; }
private string logCnString { get; set; }
private List<Log> Logs { get; set; }
public Worker(string threadPoolId)
{
this.Logs = new List<Log>();
SqlConnectionStringBuilder cnBldr = new SqlConnectionStringBuilder();
cnBldr.DataSource = #"trgcrmqa3";
cnBldr.InitialCatalog = "Scratch";
cnBldr.IntegratedSecurity = true;
cnBldr.MultipleActiveResultSets = true;
cnBldr.Pooling = true;
dataCnString = GetConnectionStringWithWorkStationId(cnBldr.ToString(), threadPoolId);
cnBldr = new SqlConnectionStringBuilder();
cnBldr.DataSource = #"trgcrmqa3";
cnBldr.InitialCatalog = "Scratch";
cnBldr.IntegratedSecurity = true;
logCnString = GetConnectionStringWithWorkStationId(cnBldr.ToString(), string.Empty);
IsEnabled = true;
}
private string machineName { get; set; }
private string GetConnectionStringWithWorkStationId(string connectionString, string connectionPoolToken)
{
if (string.IsNullOrEmpty(machineName)) machineName = Environment.MachineName;
SqlConnectionStringBuilder cnbdlr;
try
{
cnbdlr = new SqlConnectionStringBuilder(connectionString);
}
catch
{
throw new ArgumentException("connection string was an invalid format");
}
cnbdlr.WorkstationID = machineName + connectionPoolToken;
return cnbdlr.ConnectionString;
}
public void Work()
{
int i = 0;
while (this.IsEnabled)
{
this.IsRunning = true;
try
{
Log log = new Log();
log.WorkItemId = Guid.NewGuid();
log.StartTime = DateTime.Now;
List<object> lst = new List<object>();
using (SqlConnection cn = new SqlConnection(this.dataCnString))
{
try
{
cn.Open();
using (SqlCommand cmd = new SqlCommand("Analysis.spSelectTestData", cn))
{
cmd.CommandType = System.Data.CommandType.StoredProcedure;
using (SqlDataReader dr = cmd.ExecuteReader(CommandBehavior.SequentialAccess)) // DBHelper.ExecuteReader(cn, cmd))
{
while (dr.Read())
{
CreateClaimHeader2(dr, lst);
}
dr.Close();
}
cmd.Cancel();
}
}
catch { }
finally
{
cn.Close();
}
}
log.StopTime = DateTime.Now;
log.RouteName = this.Name;
log.HostName = this.machineName;
this.Logs.Add(log);
i++;
if (i > 1000)
{
Console.WriteLine(string.Format("Thread: {0} executed {1} items.", this.Name, i));
i = 0;
}
if (this.Sleep > 0) Thread.Sleep(this.Sleep);
}
catch { }
}
this.LogMessages();
this.IsRunning = false;
}
private void CreateClaimHeader2(IDataReader reader, List<object> lst)
{
lst.Add(reader["ClaimHeaderID"]);
lst.Add(reader["ClientCode"]);
lst.Add(reader["MemberID"]);
lst.Add(reader["ProviderID"]);
lst.Add(reader["ClaimNumber"]);
lst.Add(reader["PatientAcctNumber"]);
lst.Add(reader["Source"]);
lst.Add(reader["SourceID"]);
lst.Add(reader["TotalPayAmount"]);
lst.Add(reader["TotalBillAmount"]);
lst.Add(reader["FirstDateOfService"]);
lst.Add(reader["LastDateOfService"]);
lst.Add(reader["MaxStartDateOfService"]);
lst.Add(reader["MaxValidStartDateOfService"]);
lst.Add(reader["LastUpdated"]);
lst.Add(reader["UpdatedBy"]);
}
/// <summary>
/// Toes the data table.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="data">The data.</param>
/// <returns></returns>
public DataTable ToDataTable<T>(IEnumerable<T> data)
{
PropertyDescriptorCollection props =
TypeDescriptor.GetProperties(typeof(T));
if (props == null) throw new ArgumentNullException("Table properties.");
if (data == null) throw new ArgumentNullException("data");
DataTable table = new DataTable();
for (int i = 0; i < props.Count; i++)
{
PropertyDescriptor prop = props[i];
table.Columns.Add(prop.Name, Nullable.GetUnderlyingType(prop.PropertyType) ?? prop.PropertyType);
}
object[] values = new object[props.Count];
foreach (T item in data)
{
for (int i = 0; i < values.Length; i++)
{
values[i] = props[i].GetValue(item) ?? DBNull.Value;
}
table.Rows.Add(values);
}
return table;
}
private void LogMessages()
{
using (SqlConnection cn = new SqlConnection(this.logCnString))
{
try
{
cn.Open();
DataTable dt = ToDataTable(this.Logs);
Console.WriteLine(string.Format("Logging {0} records for Thread: {1}", this.Logs.Count, this.Name));
using (SqlCommand cmd = new SqlCommand("Analysis.spInsertWorkItemRouteLog", cn))
{
cmd.CommandType = System.Data.CommandType.StoredProcedure;
cmd.Parameters.AddWithValue("#dt", dt);
cmd.ExecuteNonQuery();
}
Console.WriteLine(string.Format("Logged {0} records for Thread: {1}", this.Logs.Count, this.Name));
}
finally
{
cn.Close();
}
}
}
}
}
1.A DataReader works in a connected environment,
whereas DataSet works in a disconnected environment.
2.A DataSet represents an in-memory cache of data consisting of any number of inter related DataTable objects. A DataTable object represents a tabular block of in-memory data.
SqlDataAdapter or sqlDataReader
Difference between SqlDataAdapter or sqlDataReader ?
Ans : 1.A DataReader works in a connected environment,
whereas DataSet works in a disconnected environment.
2.A DataSet represents an in-memory cache of data consisting of any number of inter related DataTable objects. A DataTable object represents a tabular block of in-memory data.
SqlDataAdapter or sqlDataReader