I have a web app running on Azure shared web site mode. A simple method where I add items to a list and sort this list, when the list size is about 300 items, takes 0.3s on my machine and 10s after deploy (on azure machine).
Does anybody has any idea why Azure is so slow?
Is any configuration I do it wrong? I use default one but replaced FREE mode with SHARED mode because I thought this would help but it seems it does not.
UPDATE:
public ActionResult GetPosts(String selectedStreams, int implicitSelectedVisualiserId, int userId)
{
DateTime begin = DateTime.UtcNow;
List<SearchQuery> selectedSearchQueries = searchQueryRepository.GetSearchQueriesOfStreamsIds(selectedStreams == String.Empty ? new List<int>() : selectedStreams.Split(',').Select(n => int.Parse(n)).ToList());
var implicitSelectedVisualiser = VisualiserModel.ToVisualiserModel(visualiserRepository.GetVisualiser(implicitSelectedVisualiserId));
var twitterSearchQueryOfImplicitSelectedVisualiser = searchQueryRepository.GetSearchQuery(implicitSelectedVisualiser.Stream.Name, Service.Twitter, userId);
var instagramSearchQueryOfImplicitSelectedVisualiser = searchQueryRepository.GetSearchQuery(implicitSelectedVisualiser.Stream.Name, Service.Instagram, userId);
var facebookSearchQueryOfImplicitSelectedVisualiser = searchQueryRepository.GetSearchQuery(implicitSelectedVisualiser.Stream.Name, Service.Facebook, userId);
var manualSearchQueryOfImplicitSelectedVisualiser = searchQueryRepository.GetSearchQuery(implicitSelectedVisualiser.Stream.Name, Service.Manual, userId);
List<SearchResultModel> approvedSearchResults = new List<SearchResultModel>();
if (twitterSearchQueryOfImplicitSelectedVisualiser != null || instagramSearchQueryOfImplicitSelectedVisualiser != null || facebookSearchQueryOfImplicitSelectedVisualiser != null
|| manualSearchQueryOfImplicitSelectedVisualiser != null)
{
// Define search text to be displayed during slideshow;
SearchModel searchModel = new SearchModel();
// Set slideshow settings from implicit selected visualiser.
ViewBag.CurrentVisualiser = implicitSelectedVisualiser;
// Load search results from selected visualisers.
foreach (SearchQuery searchQuery in selectedSearchQueries)
{
approvedSearchResults.AddRange(
SearchResultModel.ToSearchResultModel(
searchResultRepository.GetSearchResults
(searchQuery.Id,
implicitSelectedVisualiser.Language)));
// Add defined query too.
searchModel.SearchValue += " " + searchQuery.Query;
}
// Add defined query for implicit selected visualiser.
if (twitterSearchQueryOfImplicitSelectedVisualiser != null)
searchModel.SearchValue += " " + twitterSearchQueryOfImplicitSelectedVisualiser.Query;
if (instagramSearchQueryOfImplicitSelectedVisualiser != null)
searchModel.SearchValue += " " + instagramSearchQueryOfImplicitSelectedVisualiser.Query;
if (facebookSearchQueryOfImplicitSelectedVisualiser != null)
searchModel.SearchValue += " " + facebookSearchQueryOfImplicitSelectedVisualiser.Query;
ViewBag.Search = searchModel;
// Also add search results from implicit selected visualiser
if (twitterSearchQueryOfImplicitSelectedVisualiser != null)
approvedSearchResults.AddRange(SearchResultModel.ToSearchResultModel(searchResultRepository.GetSearchResults(twitterSearchQueryOfImplicitSelectedVisualiser.Id, implicitSelectedVisualiser.Language)));
if (instagramSearchQueryOfImplicitSelectedVisualiser != null)
approvedSearchResults.AddRange(SearchResultModel.ToSearchResultModel(searchResultRepository.GetSearchResults(instagramSearchQueryOfImplicitSelectedVisualiser.Id, implicitSelectedVisualiser.Language)));
if (facebookSearchQueryOfImplicitSelectedVisualiser != null)
approvedSearchResults.AddRange(SearchResultModel.ToSearchResultModel(searchResultRepository.GetSearchResults(facebookSearchQueryOfImplicitSelectedVisualiser.Id, implicitSelectedVisualiser.Language)));
if (manualSearchQueryOfImplicitSelectedVisualiser != null)
approvedSearchResults.AddRange(SearchResultModel.ToSearchResultModel(searchResultRepository.GetSearchResults(manualSearchQueryOfImplicitSelectedVisualiser.Id, implicitSelectedVisualiser.Language)));
// if user selected to show only posts from specific number of last days.
var approvedSearchResultsFilteredByDays = new List<SearchResultModel>();
if (implicitSelectedVisualiser.ShowPostsFromLastXDays != 0)
{
foreach (SearchResultModel searchResult in approvedSearchResults)
{
var postCreatedTimeWithDays = searchResult.PostCreatedTime.AddDays(implicitSelectedVisualiser.ShowPostsFromLastXDays + 1);
if (postCreatedTimeWithDays >= DateTime.Now)
approvedSearchResultsFilteredByDays.Add(searchResult);
}
}
else
{
approvedSearchResultsFilteredByDays = approvedSearchResults;
}
// Order search results (posts to be displayed by created datetime).
var approvedSearchResultsOrdered = new List<SearchResultModel>();
if (implicitSelectedVisualiser.PostsSortOrder == PostsSortOrder.CREATED_DATE_ASC)
{
approvedSearchResultsOrdered = approvedSearchResultsFilteredByDays.OrderBy(s => s.PostCreatedTime).ToList(); ;
}
else if (implicitSelectedVisualiser.PostsSortOrder == PostsSortOrder.CREATED_DATE_DESC)
{
approvedSearchResultsOrdered = approvedSearchResultsFilteredByDays.OrderByDescending(s => s.PostCreatedTime).ToList(); ;
}
else if (implicitSelectedVisualiser.PostsSortOrder == PostsSortOrder.RANDOM)
{
var rnd = new Random();
approvedSearchResultsOrdered = approvedSearchResultsFilteredByDays.OrderBy(x => rnd.Next()).ToList();
}
// Load background images;
var visualiserImages = visualiserImageRepository.GetImages(implicitSelectedVisualiser.Id);
//foreach (SearchResultModel searchResultModel in approvedSearchResultsOrdered)
//{
// searchResultModel.BackgroundImagePath = TwitterUtils.GetRandomImageBackgroundForDisplay(visualiserImages);
//}
ViewBag.BackgroundImagePath = TwitterUtils.GetRandomImageBackgroundForDisplay(visualiserImages);
approvedSearchResults = approvedSearchResultsOrdered;
}
DateTime end = DateTime.UtcNow;
Elmah.ErrorSignal.FromCurrentContext().Raise(new Exception(String.Format("User {0}: Preparing {1} posts for visualiser took {2} seconds", MySession.Current.LoggedInUserName, approvedSearchResults.Count(), (end - begin).TotalMilliseconds / 1000)));
return PartialView("_DisplayPostsNew", approvedSearchResults);
}
This isn't surprising actually. The servers used in Windows Azure are currently mostly 1.6 GHz machines. The larger sized machine you use the more cores you get, but they are all the same speed. This likely is a much slower CPU than the development machine you use.
On Windows Azure Web Sites when you move to Shared mode you are still in a multi-tenant environment, so you could be seeing some noisy neighbors here. The difference between Free and Shared is that many of the quotas for free are removed since you are paying. When you move to Standard then you are assigned a Virtual Machine dedicated to your web sites (up to 100 of them), so that is the best case scenario since you are the only one using the resources at that point.
There was a thread on this on the MSDN forums a while back : http://social.msdn.microsoft.com/Forums/windowsazure/en-US/0d0a3a88-eac4-4b9e-8b10-4a547cbf653b/performance-of-azure-servers-slow-cpus?forum=windowsazuredevelopment
They have started offering different hardware configurations with more memory for Virtual Machines and Cloud Services and such, but I'm not sure the CPUs have been changed. It's hard to find the CPU stated on WindowsAzure.com anymore, but on the pricing calculator for Web Sites it references 1.6Ghz machines when you move the slider to Standard.
Actually I found the issue.
Locally, I tested with a few hundreds of records in my DB while in Azure DB I have over 70 000 records in that table which affects performance of the algorithm...
One mistake I did in the code above: I have filtered records from DB by specific date AFTER taking all out. By filtering directly in Linq, I increased the performance from 10s to 0.3s in Azure too.
Related
I work on an electron application, and we would like to support dragging and dropping .msg files from our app into Outlook (or the Windows shell, or wherever, but primarily Outlook). The files will not necessarily be available before the user starts the drag and drop operation, and will need to be downloaded once the user begins the drag and drop. This means we can't use electron's inbuilt startDrag as it requires that the files are already in the filesystem, so I have implemented a node c++ addon to handle it using the OLE Drag and Drop API.
The data extraction (from my app to Outlook) needs to occur asynchronously, otherwise node's event loop will be blocked, halting the download. This means that the data extraction has to occur on another thread so that the event loop can continue actually downloading the data (currently doing the download in c++ is not an option, it has to occur in node). So I have implemented IDataObjectAsyncCapability on my DataObject, to indicate that I support asynchronous data extraction (it would be even nicer to be able to indicate that I don't support synchronous), but Outlook won't even query for the Interface, let alone do the data extraction asynchronously. The Windows shell does however query for the interface, but after the call to GetData, it still performs the extraction synchronously inside DoDragDrop, without even invoking any of the IDataObjectAsyncCapability methods. Is there something lacking in my implementation, or a specific way to encourage the drop target to start up a new thread?
Here is the meat of my DataObject implementation, if there's anything else that could be of use, I can provide.
STDMETHODIMP DataObject::QueryInterface(REFIID iid, LPVOID* ppvObject) {
if (ppvObject == nullptr) {
return E_INVALIDARG;
} else if (iid == IID_IUnknown) {
AddRef();
*ppvObject = reinterpret_cast<LPUNKNOWN>(this);
return S_OK;
} else if (iid == IID_IDataObject) {
AddRef();
*ppvObject = reinterpret_cast<LPDATAOBJECT>(this);
return S_OK;
} else if (iid == IID_IDataObjectAsyncCapability) {
AddRef();
*ppvObject = reinterpret_cast<IDataObjectAsyncCapability*>(this);
return S_OK;
}
*ppvObject = nullptr;
return E_NOINTERFACE;
}
STDMETHODIMP DataObject::GetData(LPFORMATETC queryFormat,
LPSTGMEDIUM outputStorage) {
if (queryFormat->cfFormat == CF_FILEDESCRIPTOR &&
(queryFormat->tymed & TYMED_HGLOBAL) && queryFormat->lindex <= 0 &&
queryFormat->dwAspect == DVASPECT_CONTENT) {
outputStorage->tymed = TYMED_HGLOBAL;
outputStorage->pUnkForRelease = nullptr;
outputStorage->hGlobal =
lockAndDuplicateHGLOBAL(this->groupDescriptorStorage.hGlobal);
return S_OK;
} else if (queryFormat->cfFormat == CF_FILECONTENTS &&
(queryFormat->tymed & TYMED_ISTREAM) &&
queryFormat->dwAspect == DVASPECT_CONTENT &&
queryFormat->lindex >= 0 &&
queryFormat->lindex < this->files.size()) {
// files is vector<pair<FILEDESCRIPTOR, STGMEDIUM>>
// where the STGMEDIUM is set to IStream
// Am I doing something wrong here?
auto file = this->files[queryFormat->lindex].second;
*outputStorage = file;
return S_OK;
}
return DV_E_FORMATETC;
}
STDMETHODIMP DataObject::QueryGetData(LPFORMATETC queryFormat) {
if (queryFormat->cfFormat == CF_FILEDESCRIPTOR ||
(queryFormat->cfFormat == CF_FILECONTENTS &&
(queryFormat->tymed & TYMED_HGLOBAL))) {
return S_OK;
}
return DATA_E_FORMATETC;
}
STDMETHODIMP DataObject::EnumFormatEtc(DWORD dwDirection,
LPENUMFORMATETC* ppEnumFormatEtc) {
if (dwDirection == DATADIR_GET) {
// basic implementation of IEnumFormatEtc (not mine)
return EnumFormat::Create(this->supportedFormats, this->numFormats,
ppEnumFormatEtc);
}
return E_INVALIDARG;
}
// Do the actual drag drop
// files is a list of STGMEDIUMS with IStreams. I'm using data that's readily available while
// implementing this, could that be somehow interfering and causing this problem in the first place?
DWORD dwEffect;
auto dataObject = new DataObject(groupDescriptorStorage, files);
auto dropSource = new DropSource();
// Do these DROPEFFECTS have an effect?
auto result = DoDragDrop(dataObject, dropSource,
DROPEFFECT_COPY | DROPEFFECT_MOVE, &dwEffect);
// By the time we get here, the data extraction would already have occured
// because it's happening inside DoDragDrop. of course, isAsyncOperation will be false
BOOL isInAsyncOperation;
dataObject->InOperation(&isInAsyncOperation);
I've followed the instructions from here as well as various forum posts I've found, but I can't seem to find anything about what to do when the drop target doesn't play ball.
I'm really stuck for ideas as to how to make this work. If the drop target isn't even querying for IDataObjectAsyncCapability then is it hopeless? Should I be returning something other than the file contents when it's first requested?
I currently loop through the list of google drive folders and return them all with no issues using the following...
FilesResource.ListRequest listRequest = drive.Files.List();
listRequest.Q = "mimeType='application/vnd.google-apps.folder'";
listRequest.PageSize = 1000;
listRequest.Fields = "nextPageToken, files(mimeType, id, name, parents)";
FileList listResults = null;
do
{
if (taskInfo.Cancel.IsCancellationRequested) return;
listResults = listRequest.Execute();
IList<Google.Apis.Drive.v3.Data.File> gObjects = listResults.Files;
if (gObjects != null && gObjects.Count > 0)
{
Debug.WriteLine("Got folder batch of " + gObjects.Count + " folders.");
}
} while (listRequest.PageToken != null);
The only issue I have is I would really like to know if there was a query of some type I could use to ONLY return the COUNT of folders on my Google Drive BEFORE I start listing them in a batch. Is there one?
Try to use Google Drive API Search functionality. So if you're looking for folders you can try:
mimeType = 'application/vnd.google-apps.folder' then perform a count.
I have an item bucket with more then 30 000 items inside. What I need is to quickly search items that have particular field set to particular value, or even better is to make something like SELECT WHERE fieldValue IN (1,2,3,4) statement. Are there any ready solutions?
I searched the web and the only thing I found is "Developer's Guide to Item
Buckets and Search" but there is no code examples.
You need something like this. The Bucket item is an IIndexable so it can be searched using Sitecore 7 search API.
This code snippet below can easily be adapted to meet your needs and it's just a question of modifying the where clause.if you need any further help with the sitecore 7 syntax just write a comment on the QuickStart blog post below and I'll get back to you.
var bucketItem = Sitecore.Context.Database.GetItem(bucketPath);
if (bucketItem != null && BucketManager.IsBucket(bucketItem))
{
using (var searchContext = ContentSearchManager.GetIndex(bucketItem as IIndexable).CreateSearchContext())
{
var result = searchContext.GetQueryable<SearchResultItem().Where(x => x.Name == itemName).FirstOrDefault();
if(result != null)
Context.Item = result.GetItem();
}
}
Further reading on my blog post here:
http://coreblimey.azurewebsites.net/sitecore-7-search-quick-start-guide/
Using Sitecore Content Editor:
Go to the bucket item then In search tab, start typing the following (replace fieldname and value with actual field name and value):
custom:fieldname|value
Then hit enter, you see the result of the query, you can multiple queries at once if you want.
Using Sitecore Content Search API:
using Sitecore.ContentSearch;
using Sitecore.ContentSearch.Linq;
using Sitecore.ContentSearch.SearchTypes;
using Sitecore.ContentSearch.Linq.Utilities
ID bucketItemID = "GUID of your bucket item";
ID templateID = "Guid of your item's template under bucket";
string values = "1,2,3,4,5";
using (var context = ContentSearchManager.GetIndex("sitecore_web_index").CreateSearchContext())
{
var predicate = PredicateBuilder.True<SearchResultItem>();
predicate = PredicateBuilder.And(item => item.TemplateId == new ID(templateID)
&& item.Paths.Contains(bucketItemID));
var innerPredicate = PredicateBuilder.False<SearchResultItem>();
foreach(string val in values.Split(','))
{
innerPredicate = PredicateBuilder.False<SearchResultItem>();
innerPredicate = innerPredicate.Or(item => item["FIELDNAME"] == val);
}
predicate = predicate.And(innerPredicate);
var result = predicate.GetResults();
List<Item> ResultsItems = new List<Item>();
foreach (var hit in result.Hits)
{
Item item = hit.Document.GetItem();
if(item !=null)
{
ResultsItems .Add(item);
}
}
}
The following links can give good start with the Search API:
http://www.fusionworkshop.co.uk/news-and-insight/tech-lab/sitecore-7-search-a-quickstart-guide#.VPw8AC4kWnI
https://www.sitecore.net/learn/blogs/technical-blogs/sitecore-7-development-team/posts/2013/06/sitecore-7-poco-explained.aspx
https://www.sitecore.net/learn/blogs/technical-blogs/sitecore-7-development-team/posts/2013/05/sitecore-7-predicate-builder.aspx
Hope this helps!
1 .Hi SO, I have a created a class for fetching user's tweets from twitter with the help of screen name. My problem is I'm getting rate limit exceeded very frequently.
2 .I had created table for screen name in which I'm saving all screen names and
3 .I had created another table to store user's tweets.
Below is my Code:
public List<TwitterProfileDetails> GetAllTweets(Func<SingleUserAuthorizer> AuthenticateCredentials,string screenname)
{
List<TwitterProfileDetails> lstofTweets = new List<TwitterProfileDetails>();
TwitterProfileDetails details = new TwitterProfileDetails();
var twitterCtx = new LinqToTwitter.TwitterContext(AuthenticateCredentials());
var helpResult =
(from help in twitterCtx.Help
where help.Type == HelpType.RateLimits &&
help.Resources == "search,users,socialgraph"
select help)
.SingleOrDefault();
foreach (var category in helpResult.RateLimits)
{
Console.WriteLine("\nCategory: {0}", category.Key);
foreach (var limit in category.Value)
{
Console.WriteLine(
"\n Resource: {0}\n Remaining: {1}\n Reset: {2}\n Limit: {3}",
limit.Resource, limit.Remaining, limit.Reset, limit.Limit);
}
}
var tweets = from t in twitterCtx.Status
where t.Type == StatusType.User && t.ScreenName == screename && t.Count == 15
select t;
if (tweets != null)
{
foreach (var tweetStatus in tweets)
{
if (tweetStatus != null)
{
lstofTweets.Add(new TwitterProfileDetails { Name = tweetStatus.User.Name, ProfileImagePath = tweetStatus.User.ProfileImageUrl, Tweets = tweetStatus.Text, UserID = tweetStatus.User.Identifier.UserID, PostedDate = Convert.ToDateTime(tweetStatus.CreatedAt),ScreenName=screename });
}
}
}
return lstofTweets;
}
I am using above method has below..
foreach (var screenObj in screenName)
{
var getTweets = api.GetAllTweets(api.AuthenticateCredentials, screenObj.UserName);
foreach (var obj in getTweets)
{
using (DBcontext = new DBContext())
{
tweets.Name = obj.Name;
tweets.ProfileImage = obj.ProfileImagePath;
tweets.PostedOn = obj.PostedDate;
tweets.Tweets = obj.Tweets;
tweets.CreatedOn = DateTime.Now;
tweets.ModifiedOn = DateTime.Now;
tweets.Status = EntityStatus.Active;
tweets.ScreenName = obj.ScreenName;
var exist = context.UserTweets.Any(user => user.Tweets.Equals(obj.Tweets));
if (!exist)
context.UserTweets.Add(tweets);
context.SaveChanges();
}
}
}
I see that you found the Help/RateLimits query. There are various approaches you can take. e.g. add a delay between queries, delay the next query if the limit has been exceeded, or catch the exception and delay until the next 15 minute window.
If you want to monitor interactively, you can watch the rate limit for each query. The TwitterContext instance you use for performing the query contains RateLimitXxx properties that populate after every query. You'll need to read those values after the query, which appears to be inside your GetAllTweets method. You have to expose those values to your loop somehow, via return object, out params, static field, or whatever logic you feel is necessary.
// the first time through, you have the whole rate limit for the 15 minute window
foreach (var screenObj in screenName)
{
var getTweets = api.GetAllTweets(api.AuthenticateCredentials, screenObj.UserName);
// your processing logic ...
// assuming you have the RateLimitXxx values in scope
if (rateLimitRemaining == 0)
Thread.Sleep(CalculateRemainingMilliseconds(RateLimitReset));
}
RateLimitRemaining is how many queries you can do in the current 15 minute window and RateLimitReset is the number of epoch seconds remaining until the rate limit resets (when you can start querying again).
It would be helpful to review the Twitter docs on Rate Limiting.
For reference, here are a couple other questions that might provide more ideas:
Twitter rate limiting
Get all followers using LINQ to Twitter
I'd like to fetch top n rows from my Azure Table with a simple TableQuery. But with the code below, all rows are fetched regardless of my limit with the Take.
What am I doing wrong?
int entryLimit = 5;
var table = GetFromHelperFunc();
TableQuery<MyEntity> query = new TableQuery<MyEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "MyPK"))
.Take(entryLimit);
List<FeedEntry> entryList = new List<FeedEntry>();
TableQuerySegment<FeedEntry> currentSegment = null;
while (currentSegment == null || currentSegment.ContinuationToken != null)
{
currentSegment = table.ExecuteQuerySegmented(query, this.EntryResolver, currentSegment != null ? currentSegment.ContinuationToken : null);
entryList.AddRange(currentSegment.Results);
}
Trace.WriteLine(entryList.Count) // <-- Why does this exceed my limit?
The Take method on the storage SDK doesn't work like it would in LINQ. Imagine you do something like this:
TableQuery<TableEntity> query = new TableQuery<TableEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "temp"))
.Take(5);
var result = table.ExecuteQuery(query);
When you start iterating over result you'll initially get only 5 items. But underneath, if you keep iterating over the result, the SDK will keep querying the table (and proceed to the next 'page' of 5 items).
If I have 5000 items in my table, this code will output all 5000 items (and underneath the SDK will do 1000 requests and fetch 5 items per request):
TableQuery<TableEntity> query = new TableQuery<TableEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "temp"))
.Take(5);
var result = table.ExecuteQuery(query);
foreach (var item in result)
{
Trace.WriteLine(item.RowKey);
}
The following code will fetch exactly 5 items in 1 request and stop there:
TableQuery<TableEntity> query = new TableQuery<TableEntity>()
.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "temp"))
.Take(5);
var result = table.ExecuteQuery(query);
int index = 0;
foreach (var item in result)
{
Console.WriteLine(item.RowKey);
index++;
if (index == 5)
break;
}
Actually, the Take() method sets the page size or the "take count" (TakeCount property on TableQuery). But it's still up to you to stop iterating on time if you only want 5 records.
In your example, you should modify the while loop to stop when reaching the TakeCount (which you set by calling Take):
while (entryList.Count < query.TakeCount && (currentSegment == null || currentSegment.ContinuationToken != null))
{
currentSegment = table.ExecuteQuerySegmented(query, currentSegment != null ? currentSegment.ContinuationToken : null);
entryList.AddRange(currentSegment.Results);
}
AFAIK Storage Client Library 2.0 had a bug in Take implementation. It was fixed in ver 2.0.4.
Read last comments at http://blogs.msdn.com/b/windowsazurestorage/archive/2012/11/06/windows-azure-storage-client-library-2-0-tables-deep-dive.aspx
[EDIT]
Original MSDN post no longer available. Still present on WebArchive:
http://web.archive.org/web/20200722170914/https://learn.microsoft.com/en-us/archive/blogs/windowsazurestorage/windows-azure-storage-client-library-2-0-tables-deep-dive