My goal is a VCL app where I need to concurrently download some files (URLs stored in TDataSet), then each of downloaded file must be processed (one-by-one). During app working GUI must not be hanged, user user should be able to cancel (interrupt) process in any stage.
I totally rewrite my first example, now there is no third-party classes (Omni Thread Library 3.07.6 and VCL only). Of course, it's still demo and some checks was removed from code, but however, this sample still not short unfortunately.
Downloading part based on this answer (thanks!).
So, when any file downloading I need to show progress to this files in GUI. Downloader class generate "event" OnProgressChange (because in real app I'll use TALWinInetHTTPClient class instance from Alcinoe library and it has real event OnProgressChange). I think it's enough just write progress value in DB, then DBGrid show progress values correctly.
Of course I saw this answers (and some other related with OTL):
How to Stop all Pipeline tasks correctly
How to use Pipeline pattern in Delphi (it's similar to my tasks, but differences in details).
Also I saw OTL docs and examples but I can't find some real example for doing similar task.
I created some classes to solve this task, and it's works, but have some critical troubles:
At first stage downloading not starts separately (not parallels, but one-by-one).
Cancellation not works properly.
Below some code to illustrate my problem. It's contain two units, one is main form (GUI, preparing data, interaction with user), second is Pipeline wrapper and downloader.
DFM for main form is:
object fmMain: TfmMain
Left = 628
Top = 172
Caption = 'WorkSpace preparer'
ClientHeight = 262
ClientWidth = 700
Color = clBtnFace
Font.Charset = DEFAULT_CHARSET
Font.Color = clWindowText
Font.Height = -13
Font.Name = 'Segoe UI'
Font.Style = []
OldCreateOrder = False
Position = poScreenCenter
OnCloseQuery = FormCloseQuery
OnCreate = FormCreate
PixelsPerInch = 96
TextHeight = 17
object DBGridApps: TDBGrid
AlignWithMargins = True
Left = 3
Top = 3
Width = 694
Height = 207
Align = alClient
DataSource = dsApps
ReadOnly = True
TabOrder = 0
TitleFont.Charset = DEFAULT_CHARSET
TitleFont.Color = clWindowText
TitleFont.Height = -13
TitleFont.Name = 'Segoe UI'
TitleFont.Style = []
end
object Panel1: TPanel
AlignWithMargins = True
Left = 3
Top = 216
Width = 694
Height = 43
Align = alBottom
TabOrder = 1
object bbExit: TBitBtn
AlignWithMargins = True
Left = 549
Top = 4
Width = 141
Height = 35
Align = alRight
Caption = 'Exit'
TabOrder = 0
OnClick = bbExitClick
end
object bbCancel: TBitBtn
AlignWithMargins = True
Left = 151
Top = 4
Width = 141
Height = 35
Align = alLeft
Caption = 'Cancel'
TabOrder = 1
OnClick = bbCancelClick
ExplicitTop = 0
end
object bbStart: TBitBtn
AlignWithMargins = True
Left = 4
Top = 4
Width = 141
Height = 35
Align = alLeft
Caption = 'Start'
TabOrder = 2
OnClick = bbStartClick
end
end
object dsApps: TDataSource
DataSet = cdsApps
Left = 32
Top = 88
end
object cdsApps: TClientDataSet
Aggregates = <>
Params = <>
Left = 16
Top = 72
end
end
Main form code:
unit MainForm;
interface
uses
Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
Dialogs, DB, DBClient, Grids, DBGrids, StdCtrls, Buttons, ExtCtrls,
PipelineHolder;
type
TfmMain = class(TForm)
DBGridApps: TDBGrid;
dsApps: TDataSource;
Panel1: TPanel;
bbExit: TBitBtn;
bbCancel: TBitBtn;
bbStart: TBitBtn;
cdsApps: TClientDataSet;
procedure bbExitClick(Sender: TObject);
procedure bbCancelClick(Sender: TObject);
procedure bbStartClick(Sender: TObject);
procedure FormCloseQuery(Sender: TObject; var CanClose: Boolean);
procedure FormCreate(Sender: TObject);
private
{ Private declarations }
PH : TPipelineHolder;
procedure SwitchControlState;
public
{ Public declarations }
end;
var
fmMain: TfmMain;
implementation
{$R *.dfm}
procedure TfmMain.bbExitClick(Sender: TObject);
begin
Close;
end;
procedure TfmMain.bbCancelClick(Sender: TObject);
begin
if Assigned(PH) then
begin
SwitchControlState;
PH.Stop;
end;
end;
procedure TfmMain.bbStartClick(Sender: TObject);
begin
if not Assigned(PH) then
PH := TPipelineHolder.Create;
SwitchControlState;
PH.Make(cdsApps);
end;
procedure TfmMain.FormCloseQuery(Sender: TObject; var CanClose: Boolean);
begin
CanClose := MessageBox(0, 'Exit now?', 'Exit', MB_YESNO + MB_ICONQUESTION + MB_DEFBUTTON2 + MB_TOPMOST) = IDYES;
if CanClose then bbCancel.Click;
end;
procedure TfmMain.FormCreate(Sender: TObject);
begin
//Prepare dataset
cdsApps.Close;
With cdsApps do
begin
FieldDefs.Add('progress', ftFloat);
FieldDefs.Add('status', ftString, 30);
FieldDefs.Add('id', ftString, 30);
FieldDefs.Add('uid', ftString, 30);
FieldDefs.Add('title', ftString, 30);
FieldDefs.Add('url', ftString, 255);
FieldDefs.Add('silent_parameters', ftString, 255);
FieldDefs.Add('target_file', ftString, 255);
CreateDataSet;
LogChanges := False;
Open;
// Below you can change URL as you wish.
// For example I'll use VirtualBox distrib from this page: https://www.virtualbox.org/wiki/Downloads
// To correct progress values web-server must response with correct content-lenght values and must
// support HEAD command.
// Record 1
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be2e746ce46a1000cdc8b90';
Fields[3].AsString := 'SomeApp1';
Fields[4].AsString := 'VirtualBox 6.0.10';
Fields[5].AsString := 'https://download.virtualbox.org/virtualbox/6.0.10/VirtualBox-6.0.10-132072-Win.exe';
Fields[6].AsString := '/S';
Fields[7].AsString := '001_installer.exe';
Post;
// Record 2
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be31c63ce46a1000b268bb2';
Fields[3].AsString := 'SomeApp2';
Fields[4].AsString := 'VirtualBox 6.0.10';
Fields[5].AsString := 'https://download.virtualbox.org/virtualbox/6.0.10/VirtualBox-6.0.10-132072-Win.exe';
Fields[6].AsString := '';
Fields[7].AsString := '002_installer.exe';
Post;
// Record 3
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be3428ace46a1000b268bc0';
Fields[3].AsString := 'SomeApp3';
Fields[4].AsString := 'VirtualBox 6.0.10';
Fields[5].AsString := 'https://download.virtualbox.org/virtualbox/6.0.10/VirtualBox-6.0.10-132072-Win.exe';
Fields[6].AsString := '/VERY_SILENT';
Fields[7].AsString := '003_installer.exe';
Post;
// Record 4
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be3428ace46a1000b268bc1';
Fields[3].AsString := 'SomeApp4';
Fields[4].AsString := 'VirtualBox 6.0.10';
Fields[5].AsString := 'https://download.virtualbox.org/virtualbox/6.0.10/VirtualBox-6.0.10-132072-Win.exe';
Fields[6].AsString := '';
Fields[7].AsString := '004_installer.exe';
Post;
// Record 5 - it's not exe, just simple testing file, in this demo at
// Install method with this file will set status to error.
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be512bb4a9bbb000b6de944';
Fields[3].AsString := 'SomeFile';
Fields[4].AsString := 'Demo File (not executable)';
Fields[5].AsString := 'https://speed.hetzner.de/100MB.bin';
Fields[6].AsString := '';
Fields[7].AsString := '005_sample_100MB.bin';
Post;
First;
end;
end;
procedure TfmMain.SwitchControlState;
begin
bbStart.Enabled := not bbStart.Enabled;
end;
end.
Second unit for pipeline working implementation:
unit PipelineHolder;
interface
uses
Windows, SysUtils, Classes, OtlCommon, OtlCollections, OtlParallel, Forms,
DB, Generics.Defaults, StrUtils, Generics.Collections, Messages, OtlComm,
OtlTask, OtlTaskControl, ShellAPI, Dialogs, OtlSync, Math, WinInet;
// Messages
const
WM_PROGRESSCHANGED = WM_APP + 105;
// Process states
type
TAppState = (asReady = 0, asCancelled = 1, asError = 2, asDownloading = 3, asDownloaded = 4, asInstalling = 5, asCompleted = 6);
TAppStateNames = array[asReady..asCompleted] of string;
const
AppState: TAppStateNames = ('Ready', 'Canceled', 'Error', 'Downloading', 'Downloaded', 'Installing', 'Installed');
type
// Data structs for progress message
PProgressInfo = ^TProgressInfo;
TProgressInfo = record
Read : Int64;
Total : Int64;
ID : string;
URL : string;
end;
//Structure for record info
TRecordInfo = record
Filename: string;
URL: string;
ID: string;
Cmd : string;
end;
// Class for downloading
TDBAppItem = class
private
FHandle : HWND;
FDS : TDataSet;
FFilename: string;
FURL: string;
FId: string;
FCmd : string;
FFileSize : Int64;
FDownloaded : Int64;
function GetWinInetError(ErrorCode: Cardinal): string;
procedure ParseURL(const lpszUrl: string; var Host, Resource: string);
function GetRemoteFileSize(const Url : string): Integer;
function DownloadFile(const url: string; const TargetFileName: string): boolean;
procedure InternalDownloadProgress(Sender: TObject; Read: Integer; Total: Integer);
public
constructor Create(const OwnerHandle: HWND; var DS: TDataSet; const URL, ID: string; const Cmd: string; const TargetFilename: string);
destructor Destroy; override;
function Download : Boolean; overload;
end;
// Main class, pipeline holder
TPipelineHolder = class
private
FDS : TDataSet;
FHandle : HWND;
FPipeline : IOmniPipeline;
FInProcess: Boolean;
procedure Retrieve(const input: TOmniValue; var output: TOmniValue);
procedure RetrieveAll(const input, output: IOmniBlockingCollection);
procedure Install(const input, output: IOmniBlockingCollection);
procedure JobDone;
procedure WndProc(var Message: TMessage);
procedure WMProgressChanged(var msg: TMessage); message WM_PROGRESSCHANGED;
public
constructor Create;
destructor Destroy; override;
procedure Make(SourceDS : TDataSet);
function Stop: Boolean;
property InProcess: Boolean read FInProcess write FInProcess;
end;
implementation
{ Tools }
function RunAsAdmin(const Handle: HWnd; const Filename, Params: string): Boolean;
var
sei: TShellExecuteInfo;
begin
FillChar(sei, SizeOf(sei), 0);
sei.cbSize := SizeOf(sei);
sei.Wnd := Handle;
sei.fMask := SEE_MASK_FLAG_DDEWAIT or SEE_MASK_FLAG_NO_UI;
sei.lpVerb := 'runas';
sei.lpFile := PChar(Filename);
sei.lpParameters := PChar(Params);
sei.nShow := SW_SHOWNORMAL;
Result := ShellExecuteEx(#sei);
end;
{TPipelineHolder}
constructor TPipelineHolder.Create;
begin
inherited Create;
FHandle := AllocateHWnd(WndProc);
FInProcess := False;
end;
destructor TPipelineHolder.Destroy;
begin
if FInProcess then
if Assigned(FPipeline) then
begin
FPipeline.Cancel;
FPipeline := nil;
FInProcess := False;
end;
if FHandle <> 0 then DeallocateHWnd(FHandle);
inherited;
end;
procedure TPipelineHolder.Install(const input, output: IOmniBlockingCollection);
var
app : TOmniValue;
appFile : string;
appParams: string;
ID : string;
State : string;
AppInfo : TRecordInfo;
begin
// In real app here is downloaded file must be started as separate process and
// we must wait when it will be completed.
for app in input do
begin
if not app.IsEmpty then
begin
AppInfo := app.ToRecord<TRecordInfo>;
appFile := AppInfo.Filename;
appParams := AppInfo.Cmd;
ID := AppInfo.ID;
if (appFile <> EmptyStr) and (FileExists(appFile)) then
begin
// Change file state
FDS.DisableControls;
try
if FDS.Locate('id', ID, [loCaseInsensitive]) then
begin
FDS.Edit;
FDS.FieldByName('Status').AsString := AppState[asInstalling];
FDS.Post;
end;
finally
FDS.EnableControls;
end;
// Try to execute intsaller
if RunAsAdmin(Application.Handle, appFile, appParams) then
begin
State := AppState[asCompleted]
end
else
begin
State := AppState[asError];
end;
// Change state again
FDS.DisableControls;
try
if FDS.Locate('id', ID, [loCaseInsensitive]) then
begin
FDS.Edit;
FDS.FieldByName('Status').AsString := State;
FDS.Post;
end;
finally
FDS.EnableControls;
end;
end;
end;
end;
end;
procedure TPipelineHolder.Retrieve(const input: TOmniValue; var output: TOmniValue);
var
App: TDBAppItem;
Info : TRecordInfo;
begin
// Checking cancellation flag
if not FInProcess then Exit;
// Preparing
Info := input.ToRecord<TRecordInfo>;
App := TDBAppItem.Create(FHandle, FDS, Info.URL, Info.ID, Info.Cmd, Info.Filename);
// Downloading
try
if App.Download then
output := TOmniValue.FromRecord<TRecordInfo>(Info)
else
output.Clear;
finally
FreeAndNil(App);
end;
end;
procedure TPipelineHolder.RetrieveAll(const input,
output: IOmniBlockingCollection);
var
App: TDBAppItem;
Info : TRecordInfo;
value : TOmniValue;
begin
// Preparing
for value in input do
begin
if not FInProcess then Exit;
Info := value.ToRecord<TRecordInfo>;
App := TDBAppItem.Create(FHandle, FDS, Info.URL, Info.ID, Info.Cmd, Info.Filename);
// Downloading
try
if App.Download then
output.Add(TOmniValue.FromRecord<TRecordInfo>(Info));
finally
FreeAndNil(App);
end;
end;
end;
function TPipelineHolder.Stop: Boolean;
begin
if FInProcess then
begin
if Assigned(FPipeline) then
begin
FPipeline.Cancel;
FPipeline := nil;
FInProcess := False;
end;
end;
Result := not FInProcess;
end;
procedure TPipelineHolder.WMProgressChanged(var msg: TMessage);
var
MsgRec : TProgressInfo;
Percent, Current : Double;
Read, Total : Int64;
ID : string;
begin
MsgRec := PProgressInfo(Msg.LParam )^;
Read := MsgRec.Read;
Total := MsgRec.Total;
Percent := 100 * Read / Total;
ID := MsgRec.ID;
// Write data to db
if FDS.Locate('id', ID, [loCaseInsensitive]) then
begin
FDS.DisableControls;
try
Current := FDS.FieldByName('Progress').AsFloat;
if Current <> Trunc(Percent) then
begin
FDS.Edit;
FDS.FieldByName('Progress').AsFloat := Round(Percent);
if Percent >= 99 then
begin
FDS.FieldByName('Status').AsString := AppState[asDownloaded];
end;
FDS.Post;
end;
finally
FDS.EnableControls;
end;
end;
end;
procedure TPipelineHolder.WndProc(var Message: TMessage);
begin
Dispatch(Message);
inherited;
end;
procedure TPipelineHolder.JobDone;
begin
FPipeline := nil;
FInProcess := False;
end;
procedure TPipelineHolder.Make(SourceDS: TDataSet);
var
BM : TBookmark;
RecInfo : TRecordInfo;
begin
if SourceDS = nil then Exit;
if not SourceDS.Active then Exit;
if SourceDS.IsEmpty then Exit;
FDS := SourceDS;
FInProcess := True;
// Here at first stage calling Retrive or RetrieveAll gives same effect, no
// matter what we uses value or queue.
FPipeline := Parallel.Pipeline
.Stage(RetrieveAll, //Retrieve
Parallel.TaskConfig.OnMessage(Self)).NumTasks(Environment.Process.Affinity.Count * 2)
.Stage(Install)
.OnStop(JobDone)
.Run;
// Get URLs to be downloaded
BM := FDS.GetBookmark;
FDS.DisableControls;
try
FDS.First;
while not FDS.Eof do
begin
// Get data from database
RecInfo.URL := Trim(FDS.FieldByName('url').AsString);
RecInfo.Id := Trim(FDS.FieldByName('id').AsString);
RecInfo.Cmd := Trim(FDS.FieldByName('silent_parameters').AsString);
RecInfo.Filename := ExtractFilePath(ParamStr(0)) + 'Downloads\' + Trim(FDS.FieldByName('target_file').AsString);
if RecInfo.URL = EmptyStr then
begin
// Skips empty URLs
FDS.Next;
Continue;
end;
FDS.Edit;
FDS.FieldByName('Status').AsString := AppState[asDownloading];
FDS.Post;
FPipeline.Input.Add(TOmniValue.FromRecord<TRecordInfo>(RecInfo));
FDS.Next;
end;
finally
if FDS.BookmarkValid(BM) then SourceDS.GotoBookmark(BM);
FDS.FreeBookmark(BM);
FDS.EnableControls;
end;
FPipeline.Input.CompleteAdding;
// Wait for pipeline to complete - I'm not use it to avoid GUI freezing
// FPipeline.WaitFor(INFINITE);
end;
constructor TDBAppItem.Create(const OwnerHandle: HWND; var DS: TDataSet; const URL, ID, Cmd, TargetFilename: string);
begin
inherited Create;
FDS := DS;
FURL := URL;
FId := ID;
FCmd := Cmd;
FFilename := TargetFilename;
FHandle := OwnerHandle;
FFileSize := -1;
FDownloaded := 0;
end;
destructor TDBAppItem.Destroy;
begin
FDS := nil;
inherited;
end;
function TDBAppItem.Download: Boolean;
var
path : string;
begin
path := ExtractFilePath(FFilename);
if not DirectoryExists(path) then
if not ForceDirectories(path) then
raise Exception.Create('Cannot create directory: "'+path+'".');
if FileExists(FFilename) then
try
if not DeleteFile(FFilename) then
raise Exception.Create('Cannot delete file: "'+FFilename+'".');
except on E: Exception do
raise Exception.Create('Cannot delete file: "'+FFilename+'".'+sLineBreak + E.Message);
end;
Result := DownloadFile(FURL, FFilename);
if Result then Result := FileExists(FFilename);
end;
function TDBAppItem.DownloadFile(const url, TargetFileName: string): boolean;
var
hInet: HINTERNET;
hFile: HINTERNET;
localFile: file;
buffer: array[1..65535] of Byte;
bytesRead: DWORD;
b: boolean;
begin
b := False;
if FFileSize < 0 then FFileSize := GetRemoteFileSize(url);
FDownloaded := 0;
hInet := WinInet.InternetOpen('MyFileAgent', INTERNET_OPEN_TYPE_DIRECT, nil, nil, 0);
if Assigned(hInet) then
begin
hFile := InternetOpenURL(hInet, PChar(url), nil, 0, INTERNET_FLAG_PRAGMA_NOCACHE, 0);
if Assigned(hFile) then
begin
AssignFile(localFile, TargetFileName);
Rewrite(localFile, 1);
bytesRead := 0;
repeat
InternetReadFile(hFile, #buffer, SizeOf(buffer), bytesRead);
BlockWrite(localFile, buffer, bytesRead);
Inc(FDownloaded, bytesRead);
//In real app this event occurs in TALWinHttpClient from Alcinoe library.
InternalDownloadProgress(Self, FDownloaded, FFileSize);
until bytesRead = 0;
CloseFile(localFile);
InternetCloseHandle(hFile);
end;
InternetCloseHandle(hInet);
b := true;
end;
DownloadFile := b;
FFileSize := -1;
FDownloaded := 0;
end;
function TDBAppItem.GetRemoteFileSize(const Url: string): Integer;
const
sUserAgent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36';
var
hInet : HINTERNET;
hConnect : HINTERNET;
hRequest : HINTERNET;
lpdwBufferLength: DWORD;
lpdwReserved : DWORD;
ServerName, Resource: string;
// Prot, Host, User, Pass, Path, Extra: string;
ErrorCode : Cardinal;
begin
Result := -1;
ParseURL(Url, ServerName, Resource);
hInet := InternetOpen(PChar(sUserAgent), INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
if hInet=nil then
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('InternetOpen Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
try
hConnect := InternetConnect(hInet, PChar(ServerName), INTERNET_DEFAULT_HTTP_PORT, nil, nil, INTERNET_SERVICE_HTTP, 0, 0);
if hConnect=nil then
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('InternetConnect Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
try
hRequest := HttpOpenRequest(hConnect, PChar('HEAD'), PChar(Resource), nil, nil, nil, 0, 0);
if hRequest<>nil then
begin
try
lpdwBufferLength := SizeOf(Result);
lpdwReserved := 0;
if not HttpSendRequest(hRequest, nil, 0, nil, 0) then
begin
ErrorCode := GetLastError;
raise Exception.Create(Format('HttpOpenRequest Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
if not HttpQueryInfo(hRequest, HTTP_QUERY_CONTENT_LENGTH or HTTP_QUERY_FLAG_NUMBER, #Result, lpdwBufferLength, lpdwReserved) then
begin
Result := 0;
ErrorCode := GetLastError;
raise Exception.Create(Format('HttpQueryInfo Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
finally
InternetCloseHandle(hRequest);
end;
end
else
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('HttpOpenRequest Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
finally
InternetCloseHandle(hConnect);
end;
finally
InternetCloseHandle(hInet);
end;
end;
function TDBAppItem.GetWinInetError(ErrorCode: Cardinal): string;
const
winetdll = 'wininet.dll';
var
Len: Integer;
Buffer: PChar;
begin
Len := FormatMessage(
FORMAT_MESSAGE_FROM_HMODULE or FORMAT_MESSAGE_FROM_SYSTEM or
FORMAT_MESSAGE_ALLOCATE_BUFFER or FORMAT_MESSAGE_IGNORE_INSERTS or FORMAT_MESSAGE_ARGUMENT_ARRAY,
Pointer(GetModuleHandle(winetdll)), ErrorCode, 0, #Buffer, SizeOf(Buffer), nil);
try
while (Len > 0) and (CharInSet(Buffer[Len - 1], [#0..#32, '.'])) do Dec(Len);
SetString(Result, Buffer, Len);
finally
LocalFree(HLOCAL(Buffer));
end;
end;
procedure TDBAppItem.ParseURL(const lpszUrl: string; var Host,
Resource: string);
var
lpszScheme : array[0..INTERNET_MAX_SCHEME_LENGTH - 1] of Char;
lpszHostName : array[0..INTERNET_MAX_HOST_NAME_LENGTH - 1] of Char;
lpszUserName : array[0..INTERNET_MAX_USER_NAME_LENGTH - 1] of Char;
lpszPassword : array[0..INTERNET_MAX_PASSWORD_LENGTH - 1] of Char;
lpszUrlPath : array[0..INTERNET_MAX_PATH_LENGTH - 1] of Char;
lpszExtraInfo : array[0..1024 - 1] of Char;
lpUrlComponents : TURLComponents;
begin
ZeroMemory(#lpszScheme, SizeOf(lpszScheme));
ZeroMemory(#lpszHostName, SizeOf(lpszHostName));
ZeroMemory(#lpszUserName, SizeOf(lpszUserName));
ZeroMemory(#lpszPassword, SizeOf(lpszPassword));
ZeroMemory(#lpszUrlPath, SizeOf(lpszUrlPath));
ZeroMemory(#lpszExtraInfo, SizeOf(lpszExtraInfo));
ZeroMemory(#lpUrlComponents, SizeOf(TURLComponents));
lpUrlComponents.dwStructSize := SizeOf(TURLComponents);
lpUrlComponents.lpszScheme := lpszScheme;
lpUrlComponents.dwSchemeLength := SizeOf(lpszScheme);
lpUrlComponents.lpszHostName := lpszHostName;
lpUrlComponents.dwHostNameLength := SizeOf(lpszHostName);
lpUrlComponents.lpszUserName := lpszUserName;
lpUrlComponents.dwUserNameLength := SizeOf(lpszUserName);
lpUrlComponents.lpszPassword := lpszPassword;
lpUrlComponents.dwPasswordLength := SizeOf(lpszPassword);
lpUrlComponents.lpszUrlPath := lpszUrlPath;
lpUrlComponents.dwUrlPathLength := SizeOf(lpszUrlPath);
lpUrlComponents.lpszExtraInfo := lpszExtraInfo;
lpUrlComponents.dwExtraInfoLength := SizeOf(lpszExtraInfo);
InternetCrackUrl(PChar(lpszUrl), Length(lpszUrl), ICU_DECODE or ICU_ESCAPE, lpUrlComponents);
Host := lpszHostName;
Resource := lpszUrlPath;
end;
procedure TDBAppItem.InternalDownloadProgress(Sender: TObject; Read,
Total: Integer);
var
MsgRec : PProgressInfo;
begin
// Create progress changed message
New(MsgRec);
MsgRec^.ID := Fid;
MsgRec^.Read := Read;
MsgRec^.Total := Total;
MsgRec^.URL := FURL;
SendMessage(FHandle, WM_PROGRESSCHANGED, 0, LongInt(MsgRec));
end;
end.
My basic idea is creating pipeline with 2 stages:
Retrieve: downloading all files at the same time (threads count is constrained by NumTasks from OTL).
Install: As soon as any file be downloaded, it must be processed by this stage. Action in this stage must be one by one, i.e. only one action in same time (in real app I won't start many installers together).
I try to understand how OTL works here, but I have not many experience with this library yet.
So, dear community, how I must rewrite my code to:
Have parallel downloadings at Stage 1 (now it works one-by-one).
Have possibility to correctly stop Pipeline with GUI (now I call FPipeline.Cancel by TButton pressing and it cannot stop tasks immediately).
Sources also placed here.
Thanks in advance. I'll be glad meet any advice here.
1) Download works in parallel just fine - as far as OTL is concerned. On my machine the test app starts three parallel downloads each time I press F9. The other two downloads get stuck in the
hFile := InternetOpenURL(hInet, PChar(url), nil, 0, INTERNET_FLAG_PRAGMA_NOCACHE, 0);
call. IOW, all five downloader threads enter InternetOpenURL, but only three exit immediately and start downloading. I have no idea why (and it is related to WinINET, not OTL).
2) Cancellation doesn't work because noone tells the DownloadFile method to stop. IOmniPipeline.Cancel just calls CompleteAdding on each pipeline and tells each stage to stop processing input. It cannot stop the code which is already working on an input element (i.e. your DownloadFile method). You must do that yourself.
One way to do that is to create a global Cancel flag and change DownloadFile so that it checks whether this flag is set in the following loop:
repeat
InternetReadFile(hFile, #buffer, SizeOf(buffer), bytesRead);
BlockWrite(localFile, buffer, bytesRead);
Inc(FDownloaded, bytesRead);
//In real app this event occurs in TALWinHttpClient from Alcinoe library.
InternalDownloadProgress(Self, FDownloaded, FFileSize);
if FCancelled then break; // <-----------
until bytesRead = 0;
You could also change InternalDownloadProgress and add a var cancelled: boolean parameter which could be set in the event handler when the pipeline needs to shut down.
i am new to Threads, i have a List contains a strings. My goal is to make multiple threads do work to this List, this codes only for a single thread because i'm learning currently, however i get AV when i press start Button.
type
TDemoThread = class(TThread)
private
procedure Abort;
protected
procedure Execute; override;
public
List: TStringList;
end;
procedure TfrmMain.StartButton1Click(Sender: TObject);
var
i: integer;
List: Tstrings;
begin
for i := 0 to memo1.Lines.Count - 1 do
begin
List := TStringList.Create;
List.Add(memo1.Lines.Strings[i]);
end;
Thread := TDemoThread.Create(True);
Thread.FreeOnTerminate := True;
Thread.Start;
end;
procedure TDemoThread.Execute;
var
lHTTP: TIdHTTP;
i: integer;
X: Tstrings;
begin
inherited;
if Terminated then
Exit;
lHTTP := TIdHTTP.Create(nil);
X := TStringList.Create;
lHTTP.ReadTimeout := 30000;
lHTTP.HandleRedirects := True;
for i := 0 to List.Count - 1 do
try
X.Text := lHTTP.Get('https://instagram.com/' + List.Strings[i]);
S := ExtractDelimitedString(X.Text);
X.Clear;
TThread.Synchronize(nil,
procedure
begin
frmMain.Memo2.Lines.Add(List.Strings[i] + ' : ' + S);
end);
finally
end;
end;
Your problem is that you never assign to the List member of the thread class:
type
TDemoThread = class(TThread)
private
procedure Abort;
protected
procedure Execute; override;
public
List: TStringList; <-- never assigned to, hence always nil
end;
Hence the access violation.
It looks like you are trying to pass the contents of memo1 to the thread. I would do that like so:
type
TDemoThread = class(TThread)
private
FData: TStringList;
protected
procedure Execute; override;
public
constructor Create;
destructor Destroy; override;
end;
constructor TDemoThread.Create(Data: TStrings);
begin
inherited Create(False);
FData := TStringList.Create;
FData.Assign(Data);
FreeOnTerminate := True;
end;
destructor TDemoThread.Destroy;
begin
FData.Free;
inherited;
end;
procedure TDemoThread.Execute;
var
lHTTP: TIdHTTP;
i: integer;
X: TStrings;
begin
inherited;
if Terminated then
Exit;
lHTTP := TIdHTTP.Create(nil);
X := TStringList.Create;
lHTTP.ReadTimeout := 30000;
lHTTP.HandleRedirects := True;
for i := 0 to FData.Count - 1 do
try
X.Text := lHTTP.Get('https://instagram.com/' + FData[i]);
S := ExtractDelimitedString(X.Text);
X.Clear;
TThread.Synchronize(nil,
procedure
begin
frmMain.Memo2.Lines.Add(FData[i] + ' : ' + S);
end);
finally
end;
end;
procedure TfrmMain.StartButton1Click(Sender: TObject);
begin
TDemoThread.Create(memo1.Lines);
end;
It is pointless to create suspended and then immediately start. It is also not permitted to hold a reference to a FreeOnTerminate thread after it has started so I removed that.
The code in TDemoThread.Execute leaks, unless you are running exclusively on an ARC platform. And the try/finally is pointless. And you don't need a string list to hold a single string. Assuming you aren't using ARC it should be:
procedure TDemoThread.Execute;
var
lHTTP: TIdHTTP;
i: integer;
S: string;
begin
if Terminated then
Exit;
lHTTP := TIdHTTP.Create(nil);
try
lHTTP.ReadTimeout := 30000;
lHTTP.HandleRedirects := True;
for i := 0 to FData.Count - 1 do
begin
S := ExtractDelimitedString(lHTTP.Get('https://instagram.com/' + FData[i]));
TThread.Synchronize(nil,
procedure
begin
frmMain.Memo2.Lines.Add(FData[i] + ' : ' + S);
end);
end;
finally
lHTTP.Free;
end;
end;
Personally I'd avoid updating the form from the threads themselves. Threads are data generators here, not GUI managers. So let them separate their concerns.
I'd make all the threads accumulate the results into the same shared container and then make a GUI thread to poll that container instead. Human eyes are slow and Windows GUI is slow too, so you should not update your GUI more often than 2 or 3 times per second. It would only waste CPU load and blur the form into being unreadable.
Another thing would be to avoid using slow TStringList unless its extra functionality (which makes it slow) is required. The regular TList<string> is more than enough as a dumb container and is faster.
type
TDemoThread = class;
TfrmMain = class(TForm)
private
Fetchers: TThreadList<TDemoThread>;
Data: TThreadList<string>;
property inProcess: Boolean read ... write SetInProcess;
public
procedure AfterConstruction; override;
procedure BeforeDestruction; override;
....
end;
// this demo makes each thread per each line - that is actually a bad design
// one better use a thread pool working over the same queue and only have
// 20-40 worker threads for all the URLs
TDemoThread = class(TThread)
private
URL: string;
List: TThreadList<string>;
Tracker: TThreadList<TDemoThread>;
protected
procedure Execute; override;
end;
procedure TfrmMain.BeforeDestruction;
begin
while TThreadList.Count > 0 do
Sleep(100);
FreeAndNil( Fetchers );
Data.Free;
inherited;
end;
procedure TfrmMain.AfterConstruction;
begin
Fetchers := TThreadList<TDemoThread>.Create;
Data := TThreadList<string>.Create;
inherited;
end;
procedure TfrmMain.StartButton1Click(Sender: TObject);
var
i: integer;
List: Tstrings;
worker: TDemoThread;
URL: string;
begin
If inProcess then exit;
for URL in memo1.Lines do begin
worker := TDemoThread.Create(True);
worker.FreeOnTerminate := True;
worker.URL := URL;
worker.List := Data;
worker.Tracker := Fetchers;
Fetchers.Add( worker );
end;
InProcess := True;
for worker in Fetchers do
worker.Start;
end;
procedure TfrmMain.SetInProcess(const Value: Boolean);
begin
if Value = InProcess then exit; // form already is in this mode
FInProcess := Value;
memo1.ReadOnly := Value;
StartButton.Enabled := not Value;
if Value then begin
Memo2.Lines.Clear;
Data.Clear;
end;
Timer1.Delay := 500; // twice per second
Timer1.Enabled := Value;
If not Value then // for future optimisation - make immediate mode change
FlushData; // when last worker thread quits, no waiting for timer event
If not Value then
ShowMessage('Work complete');
end;
procedure TfrmMain.Timer1Timer(const Sender: TObject);
begin
FlushData;
if Fetchers.Count <= 0 then
InProcess := False;
end;
procedure TfrmMain.FlushData;
begin
Data.LockList; // next two operations should go as non-interruptible atom
try
Memo2.Lines.AddStrings( Data.ToArray() );
Data.Clear;
finally
Data.UnLockList;
end;
end;
procedure TDemoThread.Execute;
var
lHTTP: TIdHTTP;
begin
try
lHTTP := TIdHTTP.Create(nil);
try
lHTTP.ReadTimeout := 30000;
lHTTP.HandleRedirects := True;
S := ExtractDelimitedString( lHTTP.Get('https://instagram.com/' + URL) );
List.Add( S );
finally
lHTTP.Destroy;
end;
finally
Tracker.Remove( Self );
end;
end;
http://docwiki.embarcadero.com/Libraries/Seattle/en/System.Classes.TThreadList
http://docwiki.embarcadero.com/Libraries/Seattle/en/System.Classes.TStrings.AddStrings
PS. Personally, I'd also use OmniThreads Library, as it generally makes maintaining data-generating threads easier. For example just managing how many threads did you created becomes setting one property and determining when all threads complete their work is another oneliner. You really should not create a thousand of threads to fetch all the URLs, instead you should have 10-20 threads in a Thread Pool that would take the URLs from a Input Queue and fetch them one after another. I suggest you reading about OTL's Parallel For and Fork-Join patterns at http://otl.17slon.com/tutorials.htm - it would allow making such an application more concise and easier to write. Pipeline pattern would probably be even better match for this task - since you anyway prepare URLs list as a source collection. Half the scaffolding in StartButtonClick would be gone, and the whole TDemoThread class too.
So, I've always faced MAJOR headaches when threading in delphi xe4-6, whether it be from threads not executing, exception handling causes app crashes, or simply the on terminate method never getting called. All the workarounds I've been instructed to use have become very tedious with issues still haunting me in XE6. My code generally has looked something like this:
procedure TmLoginForm.LoginClick(Sender: TObject);
var
l:TLoginThread;
begin
SyncTimer.Enabled:=true;
l:=TLoginThread.Create(true);
l.username:=UsernameEdit.Text;
l.password:=PasswordEdit.Text;
l.FreeOnTerminate:=true;
l.Start;
end;
procedure TLoginThread.Execute;
var
Success : Boolean;
Error : String;
begin
inherited;
Success := True;
if login(USERNAME,PASSWORD) then
begin
// do another network call maybe to get dif data.
end else
begin
Success := False;
Error := 'Login Failed. Check User/Pass combo.';
end;
Synchronize(
procedure
if success = true then
begin
DifferentForm.Show;
end else
begin
ShowMessage('Error: '+SLineBreak+Error);
end;
SyncTimer.Enabled := False;
end);
end;
And then I came across this unit from the samples in Delphi and from the forums:
unit AnonThread;
interface
uses
System.Classes, System.SysUtils, System.Generics.Collections;
type
EAnonymousThreadException = class(Exception);
TAnonymousThread<T> = class(TThread)
private
class var
CRunningThreads:TList<TThread>;
private
FThreadFunc: TFunc<T>;
FOnErrorProc: TProc<Exception>;
FOnFinishedProc: TProc<T>;
FResult: T;
FStartSuspended: Boolean;
private
procedure ThreadTerminate(Sender: TObject);
protected
procedure Execute; override;
public
constructor Create(AThreadFunc: TFunc<T>; AOnFinishedProc: TProc<T>;
AOnErrorProc: TProc<Exception>; ACreateSuspended: Boolean = False;
AFreeOnTerminate: Boolean = True);
class constructor Create;
class destructor Destroy;
end;
implementation
{$IFDEF MACOS}
uses
{$IFDEF IOS}
iOSapi.Foundation
{$ELSE}
MacApi.Foundation
{$ENDIF IOS}
;
{$ENDIF MACOS}
{ TAnonymousThread }
class constructor TAnonymousThread<T>.Create;
begin
inherited;
CRunningThreads := TList<TThread>.Create;
end;
class destructor TAnonymousThread<T>.Destroy;
begin
CRunningThreads.Free;
inherited;
end;
constructor TAnonymousThread<T>.Create(AThreadFunc: TFunc<T>; AOnFinishedProc: TProc<T>;
AOnErrorProc: TProc<Exception>; ACreateSuspended: Boolean = False; AFreeOnTerminate: Boolean = True);
begin
FOnFinishedProc := AOnFinishedProc;
FOnErrorProc := AOnErrorProc;
FThreadFunc := AThreadFunc;
OnTerminate := ThreadTerminate;
FreeOnTerminate := AFreeOnTerminate;
FStartSuspended := ACreateSuspended;
//Store a reference to this thread instance so it will play nicely in an ARC
//environment. Failure to do so can result in the TThread.Execute method
//not executing. See http://qc.embarcadero.com/wc/qcmain.aspx?d=113580
CRunningThreads.Add(Self);
inherited Create(ACreateSuspended);
end;
procedure TAnonymousThread<T>.Execute;
{$IFDEF MACOS}
var
lPool: NSAutoreleasePool;
{$ENDIF}
begin
{$IFDEF MACOS}
//Need to create an autorelease pool, otherwise any autorelease objects
//may leak.
//See https://developer.apple.com/library/ios/#documentation/Cocoa/Conceptual/MemoryMgmt/Articles/mmAutoreleasePools.html#//apple_ref/doc/uid/20000047-CJBFBEDI
lPool := TNSAutoreleasePool.Create;
try
{$ENDIF}
FResult := FThreadFunc;
{$IFDEF MACOS}
finally
lPool.drain;
end;
{$ENDIF}
end;
procedure TAnonymousThread<T>.ThreadTerminate(Sender: TObject);
var
lException: Exception;
begin
try
if Assigned(FatalException) and Assigned(FOnErrorProc) then
begin
if FatalException is Exception then
lException := Exception(FatalException)
else
lException := EAnonymousThreadException.Create(FatalException.ClassName);
FOnErrorProc(lException)
end
else if Assigned(FOnFinishedProc) then
FOnFinishedProc(FResult);
finally
CRunningThreads.Remove(Self);
end;
end;
end.
Why is that this anon thread unit above works flawlessly 100% of the time and my code crashes sometimes? For example, I can exec the same thread 6 times in a row, but then maybe on the 7th (or the first for that matter) time it causes the app to crash. No exceptions ever come up when debugging so I dont have a clue where to start fixing the issue. Also, why is it that I need a separate timer that calls "CheckSynchronize" for my code in order to GUI updates to happen but it is not needed when I use the anon thread unit?
Maybe someone can point me in the right direction to ask this question elsewhere if here is not the place. Sorry, I'm diving into documentation already, trying my best to understand.
Here is an example of a thread that may work 20 times in a row, but then randomly cause app to crash
inherited;
try
SQL:= 'Some SQL string';
if GetSQL(SQL,XMLData) then
synchronize(
procedure
var
i:Integer;
begin
try
mTasksForm.TasksListView.BeginUpdate;
if mTasksForm.TasksListView.Items.Count>0 then
mTasksForm.TasksListView.Items.Clear;
XMLDocument := TXMLDocument.Create(nil);
XMLDocument.Active:=True;
XMLDocument.Version:='1.0';
XMLDocument.LoadFromXML(XMLData);
XMLNode:=XMLDocument.DocumentElement.ChildNodes['Record'];
i:=0;
if XMLNode.ChildNodes['ID'].Text <>'' then
while XMLNode <> nil do
begin
LItem := mTasksForm.TasksListView.Items.AddItem;
with LItem do
begin
Text := XMLNode.ChildNodes['LOCATION'].Text;
Detail := XMLNode.ChildNodes['DESC'].Text +
SLineBreak+
'Assigned To: '+XMLNode.ChildNodes['NAME'].Text
tag := StrToInt(XMLNode.ChildNodes['ID'].Text);
color := TRectangle.Create(nil);
with color do
begin
if XMLNode.ChildNodes['STATUS'].Text = STATUS_DONE then
fill.Color := TAlphaColors.Lime
else if XMLNode.ChildNodes['STATUS'].Text = STATUS_OK then
fill.Color := TAlphaColors.Yellow
else
fill.Color := TAlphaColors.Crimson;
stroke.Color := fill.Color;
ButtonText := XMLNode.ChildNodes['STATUS'].Text;
end;
Bitmap := Color.MakeScreenshot;
end;
XMLNode:=XMLNode.NextSibling;
end;
finally
mTasksForm.TasksListView.EndUpdate;
for i := 0 to mTasksForm.TasksListView.Controls.Count-1 do
begin
if mTasksForm.TasksListView.Controls[I].ClassType = TSearchBox then
begin
SearchBox := TSearchBox(mTasksForm.TasksListView.Controls[I]);
Break;
end;
end;
SearchBox.Text:=' ';
SearchBox.text := ''; //have in here because if the searchbox has text, when attempting to add items then app crashes
end;
end)
else
error := 'Please check internet connection.';
finally
synchronize(
procedure
begin
if error <> '' then
ShowMessage('Erorr: '+error);
mTasksForm.Spinner.Visible:=false;
mTasksForm.SyncTimer.Enabled:=false;
end);
end;
end;
here is the GETSQL method
function GetSQL(SQL:String;var XMLData:String):Boolean;
var
PostResult,
ReturnCode : String;
PostData : TStringList;
IdHTTP : TIdHTTP;
XMLDocument : IXMLDocument;
XMLNode : IXMLNode;
Test : String;
begin
Result:=False;
XMLData:='';
XMLDocument:=TXMLDocument.Create(nil);
IdHTTP:=TIdHTTP.Create(nil);
PostData:=TStringList.Create;
PostData.Add('session='+SessionID);
PostData.Add('database='+Encode(DATABASE,''));
PostData.Add('sql='+Encode(SQL,''));
IdHTTP.Request.ContentEncoding:='UTF-8';
IdHTTP.Request.ContentType:='application/x-www-form-urlencoded';
IdHTTP.ConnectTimeout:=100000;
IdHTTP.ReadTimeout:=1000000;
try
PostResult:=IdHTTP.Post(SERVER_URL+GET_METHOD,PostData);
XMLDocument.Active:=True;
XMLDocument.Version:='1.0';
test := Decode(PostResult,'');
XMLDocument.LoadFromXML(Decode(PostResult,''));
XMLNode:=XMLDocument.DocumentElement;
try
ReturnCode:=XMLNode.ChildNodes['status'].Text;
except
ReturnCode:='200';
end;
if ReturnCode='' then begin
ReturnCode:='200';
end;
if ReturnCode='200' then begin
Result:=True;
XMLData:=Decode(PostResult,'');
end;
except
on E: Exception do begin
result:=false;
end;
end;
PostData.Free;
IdHTTP.Free;
end;
I'm writing a thread which writes event logs. When the application is closed (gracefully), I need to make sure this thread finishes its job saving the logs before it's free'd. If I call Free directly to the thread, it shouldn't immediately be destroyed, it should wait until the thread is done and there's no more work left to do.
Here is how I have my thread's execution laid out:
procedure TEventLogger.Execute;
var
L: TList;
E: PEventLog; //Custom record pointer
begin
while not Terminated do begin //Repeat continuously until terminated
try
E:= nil;
L:= LockList; //Acquire locked queue of logs to be written
try
if L.Count > 0 then begin //Check if any logs exist in queue
E:= PEventLog(L[0]); //Get next log from queue
L.Delete(0); //Remove log from queue
end;
finally
UnlockList;
end;
if E <> nil then begin
WriteEventLog(E); //Actual call to save log
end;
except
//Handle exception...
end;
Sleep(1);
end;
end;
And here's the destructor...
destructor TEventLogger.Destroy;
begin
ClearQueue; //I'm sure this should be removed
FQueue.Free;
DeleteCriticalSection(FListLock);
inherited;
end;
Now I already know that at the time when Free is called, I should raise a flag making it impossible to add any more logs to the queue - it just needs to finish what's already there. My issue is that I know the above code will forcefully be cut off when the thread is free'd.
How should I make this thread finish its work when Free has been called? Or if that's not possible, how in general should this thread be structured for this to happen?
If I call Free directly to the thread, it shouldn't immediately be destroyed, it should wait until the thread is done and there's no more work left to do.
I think you have a slight mis-understanding of what happens when you destroy a thread. When you call Free on a TThread, the following happens in the destructor:
Terminate is called.
WaitFor is called.
The remainder of the thread's destructor then runs.
In other words, calling Free already does what you ask for, namely notifying the thread method that it needs to terminate, and then waiting for it to do so.
Since you are in control of the thread's Execute method, you can do as much or as little work there once you detect that the Terminated flag has been set. As Remy suggests, you could override DoTerminate and do your last pieces of work there.
For what it is worth, this is a poor way to implement a queue. That call to Sleep(1) jumps right out at me. What you need is a blocking queue. You empty the queue and then wait on an event. When the producer adds to the queue the event is signaled so that your thread can wake up.
This is my take on how to write a consumer thread. The first piece of the jigsaw is a blocking queue. Mine looks like this:
unit BlockingQueue;
interface
uses
Windows, SyncObjs, Generics.Collections;
type
TBlockingQueue<T> = class
//see Duffy, Concurrent Programming on Windows, pp248
private
FCapacity: Integer;
FQueue: TQueue<T>;
FLock: TCriticalSection;
FNotEmpty: TEvent;
function DoEnqueue(const Value: T; IgnoreCapacity: Boolean): Boolean;
public
constructor Create(Capacity: Integer=-1);//default to unbounded
destructor Destroy; override;
function Enqueue(const Value: T): Boolean;
procedure ForceEnqueue(const Value: T);
function Dequeue: T;
end;
implementation
{ TBlockingQueue<T> }
constructor TBlockingQueue<T>.Create(Capacity: Integer);
begin
inherited Create;
FCapacity := Capacity;
FQueue := TQueue<T>.Create;
FLock := TCriticalSection.Create;
FNotEmpty := TEvent.Create(nil, True, False, '');
end;
destructor TBlockingQueue<T>.Destroy;
begin
FNotEmpty.Free;
FLock.Free;
FQueue.Free;
inherited;
end;
function TBlockingQueue<T>.DoEnqueue(const Value: T; IgnoreCapacity: Boolean): Boolean;
var
WasEmpty: Boolean;
begin
FLock.Acquire;
Try
Result := IgnoreCapacity or (FCapacity=-1) or (FQueue.Count<FCapacity);
if Result then begin
WasEmpty := FQueue.Count=0;
FQueue.Enqueue(Value);
if WasEmpty then begin
FNotEmpty.SetEvent;
end;
end;
Finally
FLock.Release;
End;
end;
function TBlockingQueue<T>.Enqueue(const Value: T): Boolean;
begin
Result := DoEnqueue(Value, False);
end;
procedure TBlockingQueue<T>.ForceEnqueue(const Value: T);
begin
DoEnqueue(Value, True);
end;
function TBlockingQueue<T>.Dequeue: T;
begin
FLock.Acquire;
Try
while FQueue.Count=0 do begin
FLock.Release;
Try
FNotEmpty.WaitFor;
Finally
FLock.Acquire;
End;
end;
Result := FQueue.Dequeue;
if FQueue.Count=0 then begin
FNotEmpty.ResetEvent;
end;
Finally
FLock.Release;
End;
end;
end.
It is completely threadsafe. Any thread can enqueue. Any thread can dequeue. The dequeue function will block if the queue is empty. The queue can be operated in either bounded or unbounded modes.
Next up we need a thread that works with such a queue. The thread simply pulls jobs off the queue until it is told to terminate. My consumer thread looks like this:
unit ConsumerThread;
interface
uses
SysUtils, Classes, BlockingQueue;
type
TConsumerThread = class(TThread)
private
FQueue: TBlockingQueue<TProc>;
FQueueFinished: Boolean;
procedure SetQueueFinished;
protected
procedure TerminatedSet; override;
procedure Execute; override;
public
constructor Create(Queue: TBlockingQueue<TProc>);
end;
implementation
{ TConsumerThread }
constructor TConsumerThread.Create(Queue: TBlockingQueue<TProc>);
begin
inherited Create(False);
FQueue := Queue;
end;
procedure TConsumerThread.SetQueueFinished;
begin
FQueueFinished := True;
end;
procedure TConsumerThread.TerminatedSet;
begin
inherited;
//ensure that, if the queue is empty, we wake up the thread so that it can quit
FQueue.ForceEnqueue(SetQueueFinished);
end;
procedure TConsumerThread.Execute;
var
Proc: TProc;
begin
while not FQueueFinished do begin
Proc := FQueue.Dequeue();
Proc();
Proc := nil;//clear Proc immediately, rather than waiting for Dequeue to return since it blocks
end;
end;
end.
This has the very property that you are looking for. Namely that when the thread is destroyed, it will process all pending tasks before completing the destructor.
To see it in action, here's a short demonstration program:
unit Main;
interface
uses
Windows, SysUtils, Classes, Controls, Forms, StdCtrls,
BlockingQueue, ConsumerThread;
type
TMainForm = class(TForm)
Memo1: TMemo;
TaskCount: TEdit;
Start: TButton;
Stop: TButton;
procedure StartClick(Sender: TObject);
procedure StopClick(Sender: TObject);
private
FQueue: TBlockingQueue<TProc>;
FThread: TConsumerThread;
procedure Proc;
procedure Output(const Msg: string);
end;
implementation
{$R *.dfm}
procedure TMainForm.Output(const Msg: string);
begin
TThread.Synchronize(FThread,
procedure
begin
Memo1.Lines.Add(Msg);
end
);
end;
procedure TMainForm.Proc;
begin
Output(Format('Consumer thread ID: %d', [GetCurrentThreadId]));
Sleep(1000);
end;
procedure TMainForm.StartClick(Sender: TObject);
var
i: Integer;
begin
Memo1.Clear;
Output(Format('Main thread ID: %d', [GetCurrentThreadId]));
FQueue := TBlockingQueue<TProc>.Create;
FThread := TConsumerThread.Create(FQueue);
for i := 1 to StrToInt(TaskCount.Text) do
FQueue.Enqueue(Proc);
end;
procedure TMainForm.StopClick(Sender: TObject);
begin
Output('Stop clicked, calling thread destructor');
FreeAndNil(FThread);
Output('Thread destroyed');
FreeAndNil(FQueue);
end;
end.
object MainForm: TMainForm
Caption = 'MainForm'
ClientHeight = 560
ClientWidth = 904
object Memo1: TMemo
Left = 0
Top = 96
Width = 904
Height = 464
Align = alBottom
end
object TaskCount: TEdit
Left = 8
Top = 8
Width = 121
Height = 21
Text = '10'
end
object Start: TButton
Left = 8
Top = 48
Width = 89
Height = 23
Caption = 'Start'
OnClick = StartClick
end
object Stop: TButton
Left = 120
Top = 48
Width = 75
Height = 23
Caption = 'Stop'
OnClick = StopClick
end
end
Here is a "lazy" EventLogger thread which will save all events in the queue.
unit EventLogger;
interface
uses
Classes, SyncObjs, Contnrs;
type
TEventItem = class
TimeStamp : TDateTime;
Info : string;
end;
TEventLogger = class( TThread )
private
FStream : TStream;
FEvent : TEvent;
FQueue : TThreadList;
protected
procedure TerminatedSet; override;
procedure Execute; override;
procedure WriteEvents;
function GetFirstItem( out AItem : TEventItem ) : Boolean;
public
constructor Create; overload;
constructor Create( CreateSuspended : Boolean ); overload;
destructor Destroy; override;
procedure LogEvent( const AInfo : string );
end;
implementation
uses
Windows, SysUtils;
{ TEventLogger }
constructor TEventLogger.Create( CreateSuspended : Boolean );
begin
FEvent := TEvent.Create;
FQueue := TThreadList.Create;
inherited;
end;
constructor TEventLogger.Create;
begin
Create( False );
end;
destructor TEventLogger.Destroy;
begin
// first the inherited part
inherited;
// now freeing the internal instances
FStream.Free;
FQueue.Free;
FEvent.Free;
end;
procedure TEventLogger.Execute;
var
LFinished : Boolean;
begin
inherited;
LFinished := False;
while not LFinished do
begin
// waiting for event with 20 seconds timeout
// maybe terminated or full queue
WaitForSingleObject( FEvent.Handle, 20000 );
// thread will finished if terminated
LFinished := Terminated;
// write all events from queue
WriteEvents;
// if the thread gets terminated while writing
// it will be still not finished ... and therefor one more loop
end;
end;
function TEventLogger.GetFirstItem( out AItem : TEventItem ) : Boolean;
var
LList : TList;
begin
LList := FQueue.LockList;
try
if LList.Count > 0
then
begin
AItem := TEventItem( LList[0] );
LList.Delete( 0 );
Result := True;
end
else
Result := False;
finally
FQueue.UnlockList;
end;
end;
procedure TEventLogger.LogEvent( const AInfo : string );
var
LList : TList;
LItem : TEventItem;
begin
if Terminated
then
Exit;
LItem := TEventItem.Create;
LItem.TimeStamp := now;
LItem.Info := AInfo;
LList := FQueue.LockList;
try
LList.Add( LItem );
// if the queue is "full" we will set the event
if LList.Count > 50
then
FEvent.SetEvent;
finally
FQueue.UnlockList;
end;
end;
procedure TEventLogger.TerminatedSet;
begin
// this is called if the thread is terminated
inherited;
FEvent.SetEvent;
end;
procedure TEventLogger.WriteEvents;
var
LItem : TEventItem;
LStream : TStream;
begin
// retrieve the first event in list
while GetFirstItem( LItem ) do
try
// writing the event to a file
if not Assigned( FStream )
then
FStream := TFileStream.Create( ChangeFileExt( ParamStr( 0 ), '.log' ), fmCreate or fmShareDenyWrite );
// just a simple log row
LStream :=
TStringStream.Create(
Format(
'[%s] %s : %s',
// when it is written to file
[FormatDateTime( 'dd.mm.yyyy hh:nn:ss.zzz', now ),
// when did it happend
FormatDateTime( 'dd.mm.yyyy hh:nn:ss.zzz', LItem.TimeStamp ),
// whats about
LItem.Info] ) + sLineBreak,
TEncoding.UTF8 );
try
LStream.Seek( 0, soFromBeginning );
FStream.CopyFrom( LStream, LStream.Size );
finally
LStream.Free;
end;
finally
LItem.Free;
end;
end;
end.
Modifying your code, I would suggest checking the last queue count in the while as well, notice variable LastCount I introduced here:
procedure TEventLogger.Execute;
var
L: TList;
E: PEventLog; //Custom record pointer
LastCount: integer;
begin
LastCount:=0;//counter warning
while not (Terminated and (LastCount=0)) do begin //Repeat continuously until terminated
try
E:= nil;
L:= LockList; //Acquire locked queue of logs to be written
try
LastCount:=L.Count;
if LastCount > 0 then begin //Check if any logs exist in queue
E:= PEventLog(L[0]); //Get next log from queue
L.Delete(0); //Remove log from queue
end;
finally
UnlockList;
end;
if E <> nil then begin
WriteEventLog(E); //Actual call to save log
end;
except
//Handle exception...
end;
Sleep(1);
end;
end;
I'm trying to make some work after a response producer.
In order to not hold time responses, I trying do this work in a Thread.
This thread create a DataModule, and do certain stuff (send emails, process file operations, etc) but the execution is blocking the main thread.
I don't use variables or any other interactions with the main thread. I don't see anything that can issuing blocks.
Can anyone help?
Thread code:
unit wdm_Thread;
interface
uses
System.SysUtils, System.Classes, Web.HTTPApp, Web.HTTPProd, Web.DSProd,Data.DB, Data.Win.ADODB;
type
TwdmThread = class(TDataModule)
dbConnection: TADOConnection;
spVisualizador: TADOStoredProc;
spVisualizadorProc: TADOStoredProc;
spVisualizadorProcFim: TADOStoredProc;
spVisualizadorProcInicio: TADOStoredProc;
ppVisualizadorOn: TDataSetPageProducer;
spVisualizadorMiniProc: TADOStoredProc;
end;
colaboreThread = class(TThread)
private
FDataModule: TwdmThread;
FPreviewPath: String;
FSessionID: String;
FUniqueID: String;
FUniqueIDVersao: String;
public
procedure Execute(); override;
class procedure Thumbnail(APreviewPath, ASessionID, AUniqueID, AUniqueIDVersao: String);
end;
implementation
uses
WinAPI.Windows, WinAPI.ActiveX;
{%CLASSGROUP 'System.Classes.TPersistent'}
{$R *.dfm}
{ threadThumbnail }
procedure colaboreThread.Execute;
begin
try
FDataModule := TwdmThread.Create(nil);
with FDataModule do
begin
spVisualizador.Parameters[1].Value := FSessionID;
spVisualizador.Parameters[2].Value := FUniqueID;
spVisualizador.Parameters[3].Value := FUniqueIDVersao;
spVisualizador.Parameters[4].Value := False;
spVisualizador.Open();
if spVisualizador.RecordCount > 0 then
try
DeleteFile(PWideChar(FPreviewPath + FUniqueIDVersao + spVisualizador.FieldByName('Extensao').AsString));
CopyFile(PWideChar(spVisualizador.FieldByName('ArmazenamentoCaminho').AsString + FUniqueIDVersao), PWideChar(FPreviewPath + FUniqueIDVersao + spVisualizador.FieldByName('Extensao').AsString), true);
spVisualizadorMiniProc.Parameters[1].Value := FSessionID;
spVisualizadorMiniProc.Parameters[2].Value := FUniqueID;
spVisualizadorMiniProc.Parameters[3].Value := FUniqueIDVersao;
spVisualizadorMiniProc.Parameters[4].Value := FPreviewPath;
spVisualizadorMiniProc.ExecProc();
finally
DeleteFile(PWideChar(FPreviewPath + FUniqueIDVersao + spVisualizador.FieldByName('Extensao').AsString));
end
end;
finally
FreeAndNil(FDataModule);
end;
end;
class procedure colaboreThread.Thumbnail(APreviewPath, ASessionID, AUniqueID, AUniqueIDVersao: String);
begin
with colaboreThread.Create(True) do
begin
FreeOnTerminate := True;
FPreviewPath := APreviewPath;
FSessionID := ASessionID;
FUniqueID := AUniqueID;
FUniqueIDVersao := FUniqueIDVersao;
if FUniqueIDVersao = '' then FUniqueIDVersao := FUniqueID;
Execute();
end;
end;
initialization
CoInitializeEx(NIL, COINIT_MULTITHREADED or COINIT_SPEED_OVER_MEMORY);
end.
WebModule implementation:
...
procedure TwdmColaboreUP.wdmTopWebUploadwaiUploadAction(Sender: TObject;
Request: TWebRequest; Response: TWebResponse; var Handled: Boolean);
begin
colaboreThread.Thumbnail(FPreviewPath, FSessionID, FArquivoUniqueID, FArquivoUniqueIDVersao);
end;
...
You are calling Execute on the thread, but you should not call it directly. You start the thread using Resume (or by omitting the Suspended parameter in its constructor). Then, the Execute method will be started in the background.
If you start Execute yourself, like you do now, it is run in the main thread as any other method.
with colaboreThread.Create(True) do
begin
FreeOnTerminate := True;
FPreviewPath := APreviewPath;
FSessionID := ASessionID;
FUniqueID := AUniqueID;
FUniqueIDVersao := FUniqueIDVersao;
if FUniqueIDVersao = '' then FUniqueIDVersao := FUniqueID;
// Execute(); <--- Not this
Resume; // <--- but this
end;