Related
Hello all,
i am having problems getting the file preview (the one shown on the right side in the Windows Explorer window) for a certain file.
So far fetching the file preview works fine, but it takes a long time (between 0.5 and 2 seconds). Thus i do not want it to be executed in the main thread (as this would interrupt the program gui).
I tried to execute the file preview extraction in a worker thread, but this yields a SIGSEGV.
The call stack is also not really useful, it only shows that the exception is raised in ShellObjHelper in Line 141 (see source code below).
Source Code for main unit:
type
TThreadedImageInfo = record
fileName: String;
width: integer;
height: integer;
icon: TIcon;
image: TImage;
bmp: TBitmap;
infoOut: String;
memo: TMemo;
end;
PThreadedImageInfo = ^TThreadedImageInfo;
procedure loadThumbnailImageFromFile(aData: Pointer);
var
XtractImage: IExtractImage;
ColorDepth: integer;
Flags: DWORD;
RT: IRunnableTask;
FileName: string;
pThreadInfo: PThreadedImageInfo;
begin
pThreadInfo := PThreadedImageInfo(aData);
if assigned(pThreadInfo) then begin
FileName := pThreadInfo^.fileName;
ColorDepth := 32;
Flags := IEIFLAG_ASPECT or IEIFLAG_QUALITY or IEIFLAG_ORIGSIZE; // = 580
if FileExists(FileName) then begin
if GetExtractImageItfPtr(FileName, XTractImage) then begin
if ExtractImageGetFileThumbnail(XtractImage, pthreadinfo^.Image.Width,
pthreadinfo^.Image.Height, ColorDepth, Flags, RT, pthreadinfo^.Bmp) then begin
if (Flags and IEIFLAG_CACHE) <> 0 then
pThreadInfo^.infoOut := pThreadInfo^.infoOut + 'Extractor does not cache the thumbnail.' + #13;
if (Flags and IEIFLAG_GLEAM) <> 0 then
pThreadInfo^.infoOut := pThreadInfo^.infoOut + 'The image has a gleam.' + #13;
if (Flags and IEIFLAG_NOSTAMP) <> 0 then
pThreadInfo^.infoOut := pThreadInfo^.infoOut + 'Extractor does not want an icon stamp on the thumbnail.' + #13;
if (Flags and IEIFLAG_NOBORDER) <> 0 then
pThreadInfo^.infoOut := pThreadInfo^.infoOut + 'Extractor does not want an a border around the thumbnail.' + #13;
end else if GetFileLargeIcon(FileName, pThreadInfo^.icon) then begin
pThreadInfo^.infoOut := 'Thumbnail is not available. Default icon displayed.';
end;
end else begin
pThreadInfo^.infoOut := 'Error loading IExtractImage.';
end;
end else begin
pThreadInfo^.infoOut := 'Error: File does not exist.';
end;
end;
end;
procedure threadDone(Sender: TObject; aData: Pointer);
var
pThreadInfo: PThreadedImageInfo;
begin
pthreadInfo := PThreadedImageInfo(aData);
if assigned(pThreadInfo) then begin
if assigned(pthreadInfo^.Bmp) then begin
pthreadinfo^.Image.Picture.Assign(pthreadInfo^.Bmp);
end else if assigned(pthreadInfo^.icon) then begin
pthreadinfo^.Image.Picture.Assign(pthreadInfo^.icon);
end else begin
pThreadInfo^.Image.Picture.Assign(nil);
end;
if assigned(pThreadInfo^.memo) then
pThreadInfo^.memo.Lines.Text := pThreadInfo^.infoOut;
if assigned(pthreadInfo^.icon) then
pthreadInfo^.icon.free();
if assigned(pthreadInfo^.bmp) then
pthreadInfo^.bmp.free();
end;
dispose(pthreadinfo);
end;
procedure TForm1.Button1Click(Sender: TObject);
var
pThreadInfo: PThreadedImageInfo;
begin
new(pThreadInfo);
pThreadInfo^.fileName := Edit1.Text;
pThreadInfo^.image := Image1;
pThreadInfo^.memo := Memo1;
pThreadInfo^.icon := nil;
pThreadInfo^.bmp := nil;
pThreadInfo^.infoOut := '';
// use worker thread:
//TThread.ExecuteInThread(#loadThumbnailImageFromFile, pThreadInfo, #threadDone);
// use main thread:
loadThumbnailImageFromFile(pThreadInfo);
threadDone(nil, pThreadInfo);
end;
Source code for helper unit:
unit ShellObjHelper;
{$MODE objfpc}{$H+}
{$IFDEF VER100}{$DEFINE DELPHI3}{$ENDIF}
interface
uses
Windows, ShlObj, ActiveX, ShellAPI, Graphics, SysUtils, ComObj;
type
{ from ShlObjIdl.h }
IExtractImage = interface
['{BB2E617C-0920-11D1-9A0B-00C04FC2D6C1}']
function GetLocation(Buffer: PWideChar; BufferSize: DWORD; var Priority: DWORD; var Size: TSize;
ColorDepth: DWORD; var Flags: DWORD): HResult; stdcall;
function Extract(var BitmapHandle: HBITMAP): HResult; stdcall;
end;
IRunnableTask = interface
['{85788D00-6807-11D0-B810-00C04FD706EC}']
function Run: HResult; stdcall;
function Kill(fWait: BOOL): HResult; stdcall;
function Suspend: HResult; stdcall;
function Resume: HResult; stdcall;
function IsRunning: Longint; stdcall;
end;
const
{ from ShlObjIdl.h }
ITSAT_MAX_PRIORITY = 2;
ITSAT_MIN_PRIORITY = 1;
ITSAT_DEFAULT_PRIORITY = 0;
IEI_PRIORITY_MAX = ITSAT_MAX_PRIORITY;
IEI_PRIORITY_MIN = ITSAT_MIN_PRIORITY;
IEIT_PRIORITY_NORMAL = ITSAT_DEFAULT_PRIORITY;
IEIFLAG_ASYNC = $001; // ask the extractor if it supports ASYNC extract (free threaded)
IEIFLAG_CACHE = $002; // returned from the extractor if it does NOT cache the thumbnail
IEIFLAG_ASPECT = $004; // passed to the extractor to beg it to render to the aspect ratio of the supplied rect
IEIFLAG_OFFLINE = $008; // if the extractor shouldn't hit the net to get any content needs for the rendering
IEIFLAG_GLEAM = $010; // does the image have a gleam? this will be returned if it does
IEIFLAG_SCREEN = $020; // render as if for the screen (this is exlusive with IEIFLAG_ASPECT)
IEIFLAG_ORIGSIZE = $040; // render to the approx size passed, but crop if neccessary
IEIFLAG_NOSTAMP = $080; // returned from the extractor if it does NOT want an icon stamp on the thumbnail
IEIFLAG_NOBORDER = $100; // returned from the extractor if it does NOT want an a border around the thumbnail
IEIFLAG_QUALITY = $200; // passed to the Extract method to indicate that a slower, higher quality image is desired,
// re-compute the thumbnail
// IShellFolder methods helper
procedure ShellFolderBindToObject(const ShellFolder: IShellFolder; PIDL: PItemIDList; const riid: TGUID; out pv);
function ShellFolderGetUIObjectOf(const ShellFolder: IShellFolder; cidl: DWORD; var PIDL: PItemIDList;
riid: TGUID; out pv): Boolean;
procedure ShellFolderParseDisplayName(const ShellFolder: IShellFolder; const DisplayName: string; out PIDL: PItemIDList);
function GetExtractImageItfPtr(const FileName: string; out XtractImage: IExtractImage): Boolean;
function GetFileLargeIcon(const FileName: string; out LargeIcon: TIcon): Boolean;
function ExtractImageGetFileThumbnail(const XtractImage: IExtractImage; ImgWidth, ImgHeight, ImgColorDepth: Integer;
var Flags: DWORD; out RunnableTask: IRunnableTask; out Bmp: TBitmap): Boolean;
procedure GetShellFolderItfPtr(const FolderName: string; Malloc: IMalloc; out TargetFolder: IShellFolder);
implementation
procedure ShellFolderBindToObject(const ShellFolder: IShellFolder; PIDL: PItemIDList; const riid: TGUID; out pv);
begin
OleCheck(ShellFolder.BindToObject(PIDL, nil, riid, {$IFDEF DELPHI3}Pointer(pv){$ELSE}pv{$ENDIF}));
end;
function ShellFolderGetUIObjectOf(const ShellFolder: IShellFolder; cidl: DWORD; var PIDL: PItemIDList;
riid: TGUID; out pv): Boolean;
begin
Result := NOERROR = ShellFolder.GetUIObjectOf(0, cidl, PIDL, riid, nil, {$IFDEF DELPHI3}Pointer(pv){$ELSE}pv{$ENDIF});
end;
procedure ShellFolderParseDisplayName(const ShellFolder: IShellFolder; const DisplayName: string; out PIDL: PItemIDList);
var
Attributes, Eaten: DWORD;
begin
OleCheck(ShellFolder.ParseDisplayName(0, nil, PWideChar(WideString(DisplayName)), Eaten, PIDL, Attributes));
end;
function GetExtractImageItfPtr(const FileName: string; out XtractImage: IExtractImage): Boolean;
var
TargetFolder: IShellFolder;
FilePath: string;
ItemIDList: PItemIDList;
Malloc: IMalloc;
begin
FilePath := ExcludeTrailingBackslash(ExtractFilePath(FileName));
OleCheck(SHGetMalloc(Malloc));
GetShellFolderItfPtr(FilePath, Malloc, TargetFolder);
ShellFolderParseDisplayName(TargetFolder, ExtractFileName(FileName), ItemIDList);
try
Result := ShellFolderGetUIObjectOf(TargetFolder, 1, ItemIDList, IExtractImage, XtractImage);
finally
Malloc.Free(ItemIDList);
end;
end;
function GetFileLargeIcon(const FileName: string; out LargeIcon: TIcon): Boolean;
var
SFI: TSHFileInfo;
begin
result := SHGetFileInfo(PChar(FileName), FILE_ATTRIBUTE_ARCHIVE, SFI, sizeof(SFI), SHGFI_ICON or SHGFI_LARGEICON) <> 0;
if result then begin
LargeIcon := TIcon.Create;
LargeIcon.Handle := SFI.hIcon;
end;
end;
function ExtractImageGetFileThumbnail(const XtractImage: IExtractImage; ImgWidth, ImgHeight, ImgColorDepth: Integer;
var Flags: DWORD; out RunnableTask: IRunnableTask; out Bmp: TBitmap): Boolean;
var
Size: TSize;
Buf: array[0..MAX_PATH] of WideChar;
BmpHandle: HBITMAP;
Priority: DWORD;
GetLocationRes: HRESULT;
procedure FreeAndNilBitmap;
begin
{$IFNDEF DELPHI3}
FreeAndNil(Bmp);
{$ELSE}
Bmp.Free;
Bmp := nil;
{$ENDIF}
end;
begin
Result := False;
RunnableTask := nil;
Size.cx := ImgWidth;
Size.cy := ImgHeight;
Priority := IEIT_PRIORITY_NORMAL;
Flags := Flags or IEIFLAG_ASYNC;
////////////////////////// EXCEPTION HERE, but only when multithreading /////////////////////////////////////////////////////
GetLocationRes := XtractImage.GetLocation(Buf, sizeof(Buf), Priority, Size, ImgColorDepth, Flags);
if (GetLocationRes = NOERROR) or (GetLocationRes = E_PENDING) then begin
if GetLocationRes = E_PENDING then begin
{ if QI for IRunnableTask succeed, we can use RunnableTask
interface pointer later to kill running extraction process.
We could spawn a new thread here to extract image. }
if S_OK <> XtractImage.QueryInterface(IRunnableTask, RunnableTask) then
RunnableTask := nil;
end;
Bmp := TBitmap.Create;
try
// This could consume a long time.
// If RunnableTask is available then calling Kill() method will immediately abort the process.
OleCheck(XtractImage.Extract(BmpHandle));
Bmp.Handle := BmpHandle;
Result := True;
except
on E: EOleSysError do begin
//-------------
OutputDebugString(PChar(string(E.ClassName) + ': ' + E.Message));
//-------------
FreeAndNilBitmap;
Result := False;
end else begin
FreeAndNilBitmap;
raise;
end;
end; { try/except }
end;
end;
procedure GetShellFolderItfPtr(const FolderName: string; Malloc: IMalloc; out TargetFolder: IShellFolder);
var
DesktopFolder: IShellFolder;
ItemIDList: PItemIDList;
begin
OleCheck(SHGetDesktopFolder(DesktopFolder));
ShellFolderParseDisplayName(DesktopFolder, FolderName, ItemIDList);
try
ShellFolderBindToObject(DesktopFolder, ItemIDList, IShellFolder, TargetFolder);
finally
Malloc.Free(ItemIDList);
end;
end;
end.
The actual question(s):
Why is the image extraction working without multithreading, but failing when using a worker thread?
How can i make this work?
I already started studying this post for another solution, but i am not yet sure how to do this.
Useful Informations:
Source for helper unit code: How to retrieve the file previews used by windows explorer in Windows vista and seven?
Multithreading example: https://lazarus-ccr.sourceforge.io/docs/rtl/classes/tthread.executeinthread.html
Activating PDF preview: open Adobe Acrobat Reader -> Edit -> Preferences -> General -> check "Enable PDF thumbnail previews"
I am using Lazarus v2.0.10 r63526 on Windows 10 Pro 64 bit.
Thanks to the comment from #IInspectable, that's the hint i needed.
Solution:
Add CoInitialize before calling GetExtractImageItfPtr and add CoUninitialize after receiving the file preview, but still within the worker thread.
Ensure that CoUninitialize is called even if exceptions occur by using try and finally`.
Working source code for main unit with worker thread:
type
TThreadedImageInfo = record
fileName: String;
width: integer;
height: integer;
icon: TIcon;
image: TImage;
bmp: TBitmap;
infoOut: String;
memo: TMemo;
end;
PThreadedImageInfo = ^TThreadedImageInfo;
procedure loadThumbnailImageFromFile(aData: Pointer);
var
XtractImage: IExtractImage;
ColorDepth: integer;
Flags: DWORD;
RT: IRunnableTask;
FileName: string;
pThreadInfo: PThreadedImageInfo;
begin
pThreadInfo := PThreadedImageInfo(aData);
if assigned(pThreadInfo) then begin
FileName := pThreadInfo^.fileName;
ColorDepth := 32;
Flags := IEIFLAG_ASPECT or IEIFLAG_QUALITY or IEIFLAG_ORIGSIZE; // = 580
if FileExists(FileName) then begin
CoInitialize(nil);
try
if GetExtractImageItfPtr(FileName, XTractImage) then begin
if ExtractImageGetFileThumbnail(XtractImage, pthreadinfo^.Image.Width,
pthreadinfo^.Image.Height, ColorDepth, Flags, RT, pthreadinfo^.Bmp) then begin
if (Flags and IEIFLAG_CACHE) <> 0 then
pThreadInfo^.infoOut := pThreadInfo^.infoOut + 'Extractor does not cache the thumbnail.' + #13;
if (Flags and IEIFLAG_GLEAM) <> 0 then
pThreadInfo^.infoOut := pThreadInfo^.infoOut + 'The image has a gleam.' + #13;
if (Flags and IEIFLAG_NOSTAMP) <> 0 then
pThreadInfo^.infoOut := pThreadInfo^.infoOut + 'Extractor does not want an icon stamp on the thumbnail.' + #13;
if (Flags and IEIFLAG_NOBORDER) <> 0 then
pThreadInfo^.infoOut := pThreadInfo^.infoOut + 'Extractor does not want an a border around the thumbnail.' + #13;
end else if GetFileLargeIcon(FileName, pThreadInfo^.icon) then begin
pThreadInfo^.infoOut := 'Thumbnail is not available. Default icon displayed.';
end;
end else begin
pThreadInfo^.infoOut := 'Error loading IExtractImage.';
end;
finally
CoUninitialize;
end;
end else begin
pThreadInfo^.infoOut := 'Error: File does not exist.';
end;
end;
end;
procedure threadDone(Sender: TObject; aData: Pointer);
var
pThreadInfo: PThreadedImageInfo;
begin
pthreadInfo := PThreadedImageInfo(aData);
if assigned(pThreadInfo) then begin
if assigned(pthreadInfo^.Bmp) then begin
pthreadinfo^.Image.Picture.Assign(pthreadInfo^.Bmp);
end else if assigned(pthreadInfo^.icon) then begin
pthreadinfo^.Image.Picture.Assign(pthreadInfo^.icon);
end else begin
pThreadInfo^.Image.Picture.Assign(nil);
end;
if assigned(pThreadInfo^.memo) then
pThreadInfo^.memo.Lines.Text := pThreadInfo^.infoOut;
if assigned(pthreadInfo^.icon) then
pthreadInfo^.icon.free();
if assigned(pthreadInfo^.bmp) then
pthreadInfo^.bmp.free();
end;
dispose(pthreadinfo);
end;
procedure TForm1.Button1Click(Sender: TObject);
var
pThreadInfo: PThreadedImageInfo;
begin
new(pThreadInfo);
pThreadInfo^.fileName := Edit1.Text;
pThreadInfo^.image := Image1;
pThreadInfo^.memo := Memo1;
pThreadInfo^.icon := nil;
pThreadInfo^.bmp := nil;
pThreadInfo^.infoOut := '';
TThread.ExecuteInThread(#loadThumbnailImageFromFile, pThreadInfo, #threadDone);
end;
My goal is a VCL app where I need to concurrently download some files (URLs stored in TDataSet), then each of downloaded file must be processed (one-by-one). During app working GUI must not be hanged, user user should be able to cancel (interrupt) process in any stage.
I totally rewrite my first example, now there is no third-party classes (Omni Thread Library 3.07.6 and VCL only). Of course, it's still demo and some checks was removed from code, but however, this sample still not short unfortunately.
Downloading part based on this answer (thanks!).
So, when any file downloading I need to show progress to this files in GUI. Downloader class generate "event" OnProgressChange (because in real app I'll use TALWinInetHTTPClient class instance from Alcinoe library and it has real event OnProgressChange). I think it's enough just write progress value in DB, then DBGrid show progress values correctly.
Of course I saw this answers (and some other related with OTL):
How to Stop all Pipeline tasks correctly
How to use Pipeline pattern in Delphi (it's similar to my tasks, but differences in details).
Also I saw OTL docs and examples but I can't find some real example for doing similar task.
I created some classes to solve this task, and it's works, but have some critical troubles:
At first stage downloading not starts separately (not parallels, but one-by-one).
Cancellation not works properly.
Below some code to illustrate my problem. It's contain two units, one is main form (GUI, preparing data, interaction with user), second is Pipeline wrapper and downloader.
DFM for main form is:
object fmMain: TfmMain
Left = 628
Top = 172
Caption = 'WorkSpace preparer'
ClientHeight = 262
ClientWidth = 700
Color = clBtnFace
Font.Charset = DEFAULT_CHARSET
Font.Color = clWindowText
Font.Height = -13
Font.Name = 'Segoe UI'
Font.Style = []
OldCreateOrder = False
Position = poScreenCenter
OnCloseQuery = FormCloseQuery
OnCreate = FormCreate
PixelsPerInch = 96
TextHeight = 17
object DBGridApps: TDBGrid
AlignWithMargins = True
Left = 3
Top = 3
Width = 694
Height = 207
Align = alClient
DataSource = dsApps
ReadOnly = True
TabOrder = 0
TitleFont.Charset = DEFAULT_CHARSET
TitleFont.Color = clWindowText
TitleFont.Height = -13
TitleFont.Name = 'Segoe UI'
TitleFont.Style = []
end
object Panel1: TPanel
AlignWithMargins = True
Left = 3
Top = 216
Width = 694
Height = 43
Align = alBottom
TabOrder = 1
object bbExit: TBitBtn
AlignWithMargins = True
Left = 549
Top = 4
Width = 141
Height = 35
Align = alRight
Caption = 'Exit'
TabOrder = 0
OnClick = bbExitClick
end
object bbCancel: TBitBtn
AlignWithMargins = True
Left = 151
Top = 4
Width = 141
Height = 35
Align = alLeft
Caption = 'Cancel'
TabOrder = 1
OnClick = bbCancelClick
ExplicitTop = 0
end
object bbStart: TBitBtn
AlignWithMargins = True
Left = 4
Top = 4
Width = 141
Height = 35
Align = alLeft
Caption = 'Start'
TabOrder = 2
OnClick = bbStartClick
end
end
object dsApps: TDataSource
DataSet = cdsApps
Left = 32
Top = 88
end
object cdsApps: TClientDataSet
Aggregates = <>
Params = <>
Left = 16
Top = 72
end
end
Main form code:
unit MainForm;
interface
uses
Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
Dialogs, DB, DBClient, Grids, DBGrids, StdCtrls, Buttons, ExtCtrls,
PipelineHolder;
type
TfmMain = class(TForm)
DBGridApps: TDBGrid;
dsApps: TDataSource;
Panel1: TPanel;
bbExit: TBitBtn;
bbCancel: TBitBtn;
bbStart: TBitBtn;
cdsApps: TClientDataSet;
procedure bbExitClick(Sender: TObject);
procedure bbCancelClick(Sender: TObject);
procedure bbStartClick(Sender: TObject);
procedure FormCloseQuery(Sender: TObject; var CanClose: Boolean);
procedure FormCreate(Sender: TObject);
private
{ Private declarations }
PH : TPipelineHolder;
procedure SwitchControlState;
public
{ Public declarations }
end;
var
fmMain: TfmMain;
implementation
{$R *.dfm}
procedure TfmMain.bbExitClick(Sender: TObject);
begin
Close;
end;
procedure TfmMain.bbCancelClick(Sender: TObject);
begin
if Assigned(PH) then
begin
SwitchControlState;
PH.Stop;
end;
end;
procedure TfmMain.bbStartClick(Sender: TObject);
begin
if not Assigned(PH) then
PH := TPipelineHolder.Create;
SwitchControlState;
PH.Make(cdsApps);
end;
procedure TfmMain.FormCloseQuery(Sender: TObject; var CanClose: Boolean);
begin
CanClose := MessageBox(0, 'Exit now?', 'Exit', MB_YESNO + MB_ICONQUESTION + MB_DEFBUTTON2 + MB_TOPMOST) = IDYES;
if CanClose then bbCancel.Click;
end;
procedure TfmMain.FormCreate(Sender: TObject);
begin
//Prepare dataset
cdsApps.Close;
With cdsApps do
begin
FieldDefs.Add('progress', ftFloat);
FieldDefs.Add('status', ftString, 30);
FieldDefs.Add('id', ftString, 30);
FieldDefs.Add('uid', ftString, 30);
FieldDefs.Add('title', ftString, 30);
FieldDefs.Add('url', ftString, 255);
FieldDefs.Add('silent_parameters', ftString, 255);
FieldDefs.Add('target_file', ftString, 255);
CreateDataSet;
LogChanges := False;
Open;
// Below you can change URL as you wish.
// For example I'll use VirtualBox distrib from this page: https://www.virtualbox.org/wiki/Downloads
// To correct progress values web-server must response with correct content-lenght values and must
// support HEAD command.
// Record 1
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be2e746ce46a1000cdc8b90';
Fields[3].AsString := 'SomeApp1';
Fields[4].AsString := 'VirtualBox 6.0.10';
Fields[5].AsString := 'https://download.virtualbox.org/virtualbox/6.0.10/VirtualBox-6.0.10-132072-Win.exe';
Fields[6].AsString := '/S';
Fields[7].AsString := '001_installer.exe';
Post;
// Record 2
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be31c63ce46a1000b268bb2';
Fields[3].AsString := 'SomeApp2';
Fields[4].AsString := 'VirtualBox 6.0.10';
Fields[5].AsString := 'https://download.virtualbox.org/virtualbox/6.0.10/VirtualBox-6.0.10-132072-Win.exe';
Fields[6].AsString := '';
Fields[7].AsString := '002_installer.exe';
Post;
// Record 3
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be3428ace46a1000b268bc0';
Fields[3].AsString := 'SomeApp3';
Fields[4].AsString := 'VirtualBox 6.0.10';
Fields[5].AsString := 'https://download.virtualbox.org/virtualbox/6.0.10/VirtualBox-6.0.10-132072-Win.exe';
Fields[6].AsString := '/VERY_SILENT';
Fields[7].AsString := '003_installer.exe';
Post;
// Record 4
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be3428ace46a1000b268bc1';
Fields[3].AsString := 'SomeApp4';
Fields[4].AsString := 'VirtualBox 6.0.10';
Fields[5].AsString := 'https://download.virtualbox.org/virtualbox/6.0.10/VirtualBox-6.0.10-132072-Win.exe';
Fields[6].AsString := '';
Fields[7].AsString := '004_installer.exe';
Post;
// Record 5 - it's not exe, just simple testing file, in this demo at
// Install method with this file will set status to error.
Append;
Fields[0].AsFloat := 0.0;
Fields[1].AsString := 'Ready';
Fields[2].AsString := '5be512bb4a9bbb000b6de944';
Fields[3].AsString := 'SomeFile';
Fields[4].AsString := 'Demo File (not executable)';
Fields[5].AsString := 'https://speed.hetzner.de/100MB.bin';
Fields[6].AsString := '';
Fields[7].AsString := '005_sample_100MB.bin';
Post;
First;
end;
end;
procedure TfmMain.SwitchControlState;
begin
bbStart.Enabled := not bbStart.Enabled;
end;
end.
Second unit for pipeline working implementation:
unit PipelineHolder;
interface
uses
Windows, SysUtils, Classes, OtlCommon, OtlCollections, OtlParallel, Forms,
DB, Generics.Defaults, StrUtils, Generics.Collections, Messages, OtlComm,
OtlTask, OtlTaskControl, ShellAPI, Dialogs, OtlSync, Math, WinInet;
// Messages
const
WM_PROGRESSCHANGED = WM_APP + 105;
// Process states
type
TAppState = (asReady = 0, asCancelled = 1, asError = 2, asDownloading = 3, asDownloaded = 4, asInstalling = 5, asCompleted = 6);
TAppStateNames = array[asReady..asCompleted] of string;
const
AppState: TAppStateNames = ('Ready', 'Canceled', 'Error', 'Downloading', 'Downloaded', 'Installing', 'Installed');
type
// Data structs for progress message
PProgressInfo = ^TProgressInfo;
TProgressInfo = record
Read : Int64;
Total : Int64;
ID : string;
URL : string;
end;
//Structure for record info
TRecordInfo = record
Filename: string;
URL: string;
ID: string;
Cmd : string;
end;
// Class for downloading
TDBAppItem = class
private
FHandle : HWND;
FDS : TDataSet;
FFilename: string;
FURL: string;
FId: string;
FCmd : string;
FFileSize : Int64;
FDownloaded : Int64;
function GetWinInetError(ErrorCode: Cardinal): string;
procedure ParseURL(const lpszUrl: string; var Host, Resource: string);
function GetRemoteFileSize(const Url : string): Integer;
function DownloadFile(const url: string; const TargetFileName: string): boolean;
procedure InternalDownloadProgress(Sender: TObject; Read: Integer; Total: Integer);
public
constructor Create(const OwnerHandle: HWND; var DS: TDataSet; const URL, ID: string; const Cmd: string; const TargetFilename: string);
destructor Destroy; override;
function Download : Boolean; overload;
end;
// Main class, pipeline holder
TPipelineHolder = class
private
FDS : TDataSet;
FHandle : HWND;
FPipeline : IOmniPipeline;
FInProcess: Boolean;
procedure Retrieve(const input: TOmniValue; var output: TOmniValue);
procedure RetrieveAll(const input, output: IOmniBlockingCollection);
procedure Install(const input, output: IOmniBlockingCollection);
procedure JobDone;
procedure WndProc(var Message: TMessage);
procedure WMProgressChanged(var msg: TMessage); message WM_PROGRESSCHANGED;
public
constructor Create;
destructor Destroy; override;
procedure Make(SourceDS : TDataSet);
function Stop: Boolean;
property InProcess: Boolean read FInProcess write FInProcess;
end;
implementation
{ Tools }
function RunAsAdmin(const Handle: HWnd; const Filename, Params: string): Boolean;
var
sei: TShellExecuteInfo;
begin
FillChar(sei, SizeOf(sei), 0);
sei.cbSize := SizeOf(sei);
sei.Wnd := Handle;
sei.fMask := SEE_MASK_FLAG_DDEWAIT or SEE_MASK_FLAG_NO_UI;
sei.lpVerb := 'runas';
sei.lpFile := PChar(Filename);
sei.lpParameters := PChar(Params);
sei.nShow := SW_SHOWNORMAL;
Result := ShellExecuteEx(#sei);
end;
{TPipelineHolder}
constructor TPipelineHolder.Create;
begin
inherited Create;
FHandle := AllocateHWnd(WndProc);
FInProcess := False;
end;
destructor TPipelineHolder.Destroy;
begin
if FInProcess then
if Assigned(FPipeline) then
begin
FPipeline.Cancel;
FPipeline := nil;
FInProcess := False;
end;
if FHandle <> 0 then DeallocateHWnd(FHandle);
inherited;
end;
procedure TPipelineHolder.Install(const input, output: IOmniBlockingCollection);
var
app : TOmniValue;
appFile : string;
appParams: string;
ID : string;
State : string;
AppInfo : TRecordInfo;
begin
// In real app here is downloaded file must be started as separate process and
// we must wait when it will be completed.
for app in input do
begin
if not app.IsEmpty then
begin
AppInfo := app.ToRecord<TRecordInfo>;
appFile := AppInfo.Filename;
appParams := AppInfo.Cmd;
ID := AppInfo.ID;
if (appFile <> EmptyStr) and (FileExists(appFile)) then
begin
// Change file state
FDS.DisableControls;
try
if FDS.Locate('id', ID, [loCaseInsensitive]) then
begin
FDS.Edit;
FDS.FieldByName('Status').AsString := AppState[asInstalling];
FDS.Post;
end;
finally
FDS.EnableControls;
end;
// Try to execute intsaller
if RunAsAdmin(Application.Handle, appFile, appParams) then
begin
State := AppState[asCompleted]
end
else
begin
State := AppState[asError];
end;
// Change state again
FDS.DisableControls;
try
if FDS.Locate('id', ID, [loCaseInsensitive]) then
begin
FDS.Edit;
FDS.FieldByName('Status').AsString := State;
FDS.Post;
end;
finally
FDS.EnableControls;
end;
end;
end;
end;
end;
procedure TPipelineHolder.Retrieve(const input: TOmniValue; var output: TOmniValue);
var
App: TDBAppItem;
Info : TRecordInfo;
begin
// Checking cancellation flag
if not FInProcess then Exit;
// Preparing
Info := input.ToRecord<TRecordInfo>;
App := TDBAppItem.Create(FHandle, FDS, Info.URL, Info.ID, Info.Cmd, Info.Filename);
// Downloading
try
if App.Download then
output := TOmniValue.FromRecord<TRecordInfo>(Info)
else
output.Clear;
finally
FreeAndNil(App);
end;
end;
procedure TPipelineHolder.RetrieveAll(const input,
output: IOmniBlockingCollection);
var
App: TDBAppItem;
Info : TRecordInfo;
value : TOmniValue;
begin
// Preparing
for value in input do
begin
if not FInProcess then Exit;
Info := value.ToRecord<TRecordInfo>;
App := TDBAppItem.Create(FHandle, FDS, Info.URL, Info.ID, Info.Cmd, Info.Filename);
// Downloading
try
if App.Download then
output.Add(TOmniValue.FromRecord<TRecordInfo>(Info));
finally
FreeAndNil(App);
end;
end;
end;
function TPipelineHolder.Stop: Boolean;
begin
if FInProcess then
begin
if Assigned(FPipeline) then
begin
FPipeline.Cancel;
FPipeline := nil;
FInProcess := False;
end;
end;
Result := not FInProcess;
end;
procedure TPipelineHolder.WMProgressChanged(var msg: TMessage);
var
MsgRec : TProgressInfo;
Percent, Current : Double;
Read, Total : Int64;
ID : string;
begin
MsgRec := PProgressInfo(Msg.LParam )^;
Read := MsgRec.Read;
Total := MsgRec.Total;
Percent := 100 * Read / Total;
ID := MsgRec.ID;
// Write data to db
if FDS.Locate('id', ID, [loCaseInsensitive]) then
begin
FDS.DisableControls;
try
Current := FDS.FieldByName('Progress').AsFloat;
if Current <> Trunc(Percent) then
begin
FDS.Edit;
FDS.FieldByName('Progress').AsFloat := Round(Percent);
if Percent >= 99 then
begin
FDS.FieldByName('Status').AsString := AppState[asDownloaded];
end;
FDS.Post;
end;
finally
FDS.EnableControls;
end;
end;
end;
procedure TPipelineHolder.WndProc(var Message: TMessage);
begin
Dispatch(Message);
inherited;
end;
procedure TPipelineHolder.JobDone;
begin
FPipeline := nil;
FInProcess := False;
end;
procedure TPipelineHolder.Make(SourceDS: TDataSet);
var
BM : TBookmark;
RecInfo : TRecordInfo;
begin
if SourceDS = nil then Exit;
if not SourceDS.Active then Exit;
if SourceDS.IsEmpty then Exit;
FDS := SourceDS;
FInProcess := True;
// Here at first stage calling Retrive or RetrieveAll gives same effect, no
// matter what we uses value or queue.
FPipeline := Parallel.Pipeline
.Stage(RetrieveAll, //Retrieve
Parallel.TaskConfig.OnMessage(Self)).NumTasks(Environment.Process.Affinity.Count * 2)
.Stage(Install)
.OnStop(JobDone)
.Run;
// Get URLs to be downloaded
BM := FDS.GetBookmark;
FDS.DisableControls;
try
FDS.First;
while not FDS.Eof do
begin
// Get data from database
RecInfo.URL := Trim(FDS.FieldByName('url').AsString);
RecInfo.Id := Trim(FDS.FieldByName('id').AsString);
RecInfo.Cmd := Trim(FDS.FieldByName('silent_parameters').AsString);
RecInfo.Filename := ExtractFilePath(ParamStr(0)) + 'Downloads\' + Trim(FDS.FieldByName('target_file').AsString);
if RecInfo.URL = EmptyStr then
begin
// Skips empty URLs
FDS.Next;
Continue;
end;
FDS.Edit;
FDS.FieldByName('Status').AsString := AppState[asDownloading];
FDS.Post;
FPipeline.Input.Add(TOmniValue.FromRecord<TRecordInfo>(RecInfo));
FDS.Next;
end;
finally
if FDS.BookmarkValid(BM) then SourceDS.GotoBookmark(BM);
FDS.FreeBookmark(BM);
FDS.EnableControls;
end;
FPipeline.Input.CompleteAdding;
// Wait for pipeline to complete - I'm not use it to avoid GUI freezing
// FPipeline.WaitFor(INFINITE);
end;
constructor TDBAppItem.Create(const OwnerHandle: HWND; var DS: TDataSet; const URL, ID, Cmd, TargetFilename: string);
begin
inherited Create;
FDS := DS;
FURL := URL;
FId := ID;
FCmd := Cmd;
FFilename := TargetFilename;
FHandle := OwnerHandle;
FFileSize := -1;
FDownloaded := 0;
end;
destructor TDBAppItem.Destroy;
begin
FDS := nil;
inherited;
end;
function TDBAppItem.Download: Boolean;
var
path : string;
begin
path := ExtractFilePath(FFilename);
if not DirectoryExists(path) then
if not ForceDirectories(path) then
raise Exception.Create('Cannot create directory: "'+path+'".');
if FileExists(FFilename) then
try
if not DeleteFile(FFilename) then
raise Exception.Create('Cannot delete file: "'+FFilename+'".');
except on E: Exception do
raise Exception.Create('Cannot delete file: "'+FFilename+'".'+sLineBreak + E.Message);
end;
Result := DownloadFile(FURL, FFilename);
if Result then Result := FileExists(FFilename);
end;
function TDBAppItem.DownloadFile(const url, TargetFileName: string): boolean;
var
hInet: HINTERNET;
hFile: HINTERNET;
localFile: file;
buffer: array[1..65535] of Byte;
bytesRead: DWORD;
b: boolean;
begin
b := False;
if FFileSize < 0 then FFileSize := GetRemoteFileSize(url);
FDownloaded := 0;
hInet := WinInet.InternetOpen('MyFileAgent', INTERNET_OPEN_TYPE_DIRECT, nil, nil, 0);
if Assigned(hInet) then
begin
hFile := InternetOpenURL(hInet, PChar(url), nil, 0, INTERNET_FLAG_PRAGMA_NOCACHE, 0);
if Assigned(hFile) then
begin
AssignFile(localFile, TargetFileName);
Rewrite(localFile, 1);
bytesRead := 0;
repeat
InternetReadFile(hFile, #buffer, SizeOf(buffer), bytesRead);
BlockWrite(localFile, buffer, bytesRead);
Inc(FDownloaded, bytesRead);
//In real app this event occurs in TALWinHttpClient from Alcinoe library.
InternalDownloadProgress(Self, FDownloaded, FFileSize);
until bytesRead = 0;
CloseFile(localFile);
InternetCloseHandle(hFile);
end;
InternetCloseHandle(hInet);
b := true;
end;
DownloadFile := b;
FFileSize := -1;
FDownloaded := 0;
end;
function TDBAppItem.GetRemoteFileSize(const Url: string): Integer;
const
sUserAgent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36';
var
hInet : HINTERNET;
hConnect : HINTERNET;
hRequest : HINTERNET;
lpdwBufferLength: DWORD;
lpdwReserved : DWORD;
ServerName, Resource: string;
// Prot, Host, User, Pass, Path, Extra: string;
ErrorCode : Cardinal;
begin
Result := -1;
ParseURL(Url, ServerName, Resource);
hInet := InternetOpen(PChar(sUserAgent), INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
if hInet=nil then
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('InternetOpen Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
try
hConnect := InternetConnect(hInet, PChar(ServerName), INTERNET_DEFAULT_HTTP_PORT, nil, nil, INTERNET_SERVICE_HTTP, 0, 0);
if hConnect=nil then
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('InternetConnect Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
try
hRequest := HttpOpenRequest(hConnect, PChar('HEAD'), PChar(Resource), nil, nil, nil, 0, 0);
if hRequest<>nil then
begin
try
lpdwBufferLength := SizeOf(Result);
lpdwReserved := 0;
if not HttpSendRequest(hRequest, nil, 0, nil, 0) then
begin
ErrorCode := GetLastError;
raise Exception.Create(Format('HttpOpenRequest Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
if not HttpQueryInfo(hRequest, HTTP_QUERY_CONTENT_LENGTH or HTTP_QUERY_FLAG_NUMBER, #Result, lpdwBufferLength, lpdwReserved) then
begin
Result := 0;
ErrorCode := GetLastError;
raise Exception.Create(Format('HttpQueryInfo Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
finally
InternetCloseHandle(hRequest);
end;
end
else
begin
ErrorCode:=GetLastError;
raise Exception.Create(Format('HttpOpenRequest Error %d Description %s',[ErrorCode,GetWinInetError(ErrorCode)]));
end;
finally
InternetCloseHandle(hConnect);
end;
finally
InternetCloseHandle(hInet);
end;
end;
function TDBAppItem.GetWinInetError(ErrorCode: Cardinal): string;
const
winetdll = 'wininet.dll';
var
Len: Integer;
Buffer: PChar;
begin
Len := FormatMessage(
FORMAT_MESSAGE_FROM_HMODULE or FORMAT_MESSAGE_FROM_SYSTEM or
FORMAT_MESSAGE_ALLOCATE_BUFFER or FORMAT_MESSAGE_IGNORE_INSERTS or FORMAT_MESSAGE_ARGUMENT_ARRAY,
Pointer(GetModuleHandle(winetdll)), ErrorCode, 0, #Buffer, SizeOf(Buffer), nil);
try
while (Len > 0) and (CharInSet(Buffer[Len - 1], [#0..#32, '.'])) do Dec(Len);
SetString(Result, Buffer, Len);
finally
LocalFree(HLOCAL(Buffer));
end;
end;
procedure TDBAppItem.ParseURL(const lpszUrl: string; var Host,
Resource: string);
var
lpszScheme : array[0..INTERNET_MAX_SCHEME_LENGTH - 1] of Char;
lpszHostName : array[0..INTERNET_MAX_HOST_NAME_LENGTH - 1] of Char;
lpszUserName : array[0..INTERNET_MAX_USER_NAME_LENGTH - 1] of Char;
lpszPassword : array[0..INTERNET_MAX_PASSWORD_LENGTH - 1] of Char;
lpszUrlPath : array[0..INTERNET_MAX_PATH_LENGTH - 1] of Char;
lpszExtraInfo : array[0..1024 - 1] of Char;
lpUrlComponents : TURLComponents;
begin
ZeroMemory(#lpszScheme, SizeOf(lpszScheme));
ZeroMemory(#lpszHostName, SizeOf(lpszHostName));
ZeroMemory(#lpszUserName, SizeOf(lpszUserName));
ZeroMemory(#lpszPassword, SizeOf(lpszPassword));
ZeroMemory(#lpszUrlPath, SizeOf(lpszUrlPath));
ZeroMemory(#lpszExtraInfo, SizeOf(lpszExtraInfo));
ZeroMemory(#lpUrlComponents, SizeOf(TURLComponents));
lpUrlComponents.dwStructSize := SizeOf(TURLComponents);
lpUrlComponents.lpszScheme := lpszScheme;
lpUrlComponents.dwSchemeLength := SizeOf(lpszScheme);
lpUrlComponents.lpszHostName := lpszHostName;
lpUrlComponents.dwHostNameLength := SizeOf(lpszHostName);
lpUrlComponents.lpszUserName := lpszUserName;
lpUrlComponents.dwUserNameLength := SizeOf(lpszUserName);
lpUrlComponents.lpszPassword := lpszPassword;
lpUrlComponents.dwPasswordLength := SizeOf(lpszPassword);
lpUrlComponents.lpszUrlPath := lpszUrlPath;
lpUrlComponents.dwUrlPathLength := SizeOf(lpszUrlPath);
lpUrlComponents.lpszExtraInfo := lpszExtraInfo;
lpUrlComponents.dwExtraInfoLength := SizeOf(lpszExtraInfo);
InternetCrackUrl(PChar(lpszUrl), Length(lpszUrl), ICU_DECODE or ICU_ESCAPE, lpUrlComponents);
Host := lpszHostName;
Resource := lpszUrlPath;
end;
procedure TDBAppItem.InternalDownloadProgress(Sender: TObject; Read,
Total: Integer);
var
MsgRec : PProgressInfo;
begin
// Create progress changed message
New(MsgRec);
MsgRec^.ID := Fid;
MsgRec^.Read := Read;
MsgRec^.Total := Total;
MsgRec^.URL := FURL;
SendMessage(FHandle, WM_PROGRESSCHANGED, 0, LongInt(MsgRec));
end;
end.
My basic idea is creating pipeline with 2 stages:
Retrieve: downloading all files at the same time (threads count is constrained by NumTasks from OTL).
Install: As soon as any file be downloaded, it must be processed by this stage. Action in this stage must be one by one, i.e. only one action in same time (in real app I won't start many installers together).
I try to understand how OTL works here, but I have not many experience with this library yet.
So, dear community, how I must rewrite my code to:
Have parallel downloadings at Stage 1 (now it works one-by-one).
Have possibility to correctly stop Pipeline with GUI (now I call FPipeline.Cancel by TButton pressing and it cannot stop tasks immediately).
Sources also placed here.
Thanks in advance. I'll be glad meet any advice here.
1) Download works in parallel just fine - as far as OTL is concerned. On my machine the test app starts three parallel downloads each time I press F9. The other two downloads get stuck in the
hFile := InternetOpenURL(hInet, PChar(url), nil, 0, INTERNET_FLAG_PRAGMA_NOCACHE, 0);
call. IOW, all five downloader threads enter InternetOpenURL, but only three exit immediately and start downloading. I have no idea why (and it is related to WinINET, not OTL).
2) Cancellation doesn't work because noone tells the DownloadFile method to stop. IOmniPipeline.Cancel just calls CompleteAdding on each pipeline and tells each stage to stop processing input. It cannot stop the code which is already working on an input element (i.e. your DownloadFile method). You must do that yourself.
One way to do that is to create a global Cancel flag and change DownloadFile so that it checks whether this flag is set in the following loop:
repeat
InternetReadFile(hFile, #buffer, SizeOf(buffer), bytesRead);
BlockWrite(localFile, buffer, bytesRead);
Inc(FDownloaded, bytesRead);
//In real app this event occurs in TALWinHttpClient from Alcinoe library.
InternalDownloadProgress(Self, FDownloaded, FFileSize);
if FCancelled then break; // <-----------
until bytesRead = 0;
You could also change InternalDownloadProgress and add a var cancelled: boolean parameter which could be set in the event handler when the pipeline needs to shut down.
How can I execute an external program from a Linux console application created in Delphi 10.2 Tokyo?
What I want to do is execute a shell command with parameters like
/home/test/qrencode -o /tmp/abc.png '08154711'
I do not need the output of the program but it should be executed synchronously.
It is easy in Windows environments but as 64 bit Linux support in Delphi (after Kylix) is quite new, I could not find any hints on the Web by now.
Any tip helping me to solve that is very appreciated.
Thanks in advance!
Davids hint pointed me to an example that helped creating the solution. The most tricky part was finding out how to convert a Delphi string to a MarshaledAString as the example used a const string as argument for popen. I tested on RHEL 7.3, runs like a charm.
uses
...
System.SysUtils,
Posix.Base,
Posix.Fcntl,
...;
type
TStreamHandle = pointer;
function popen(const command: MarshaledAString; const _type: MarshaledAString): TStreamHandle; cdecl;
external libc name _PU + 'popen';
function pclose(filehandle: TStreamHandle): int32; cdecl; external libc name _PU + 'pclose';
function fgets(buffer: pointer; size: int32; Stream: TStreamHandle): pointer; cdecl; external libc name _PU + 'fgets';
function runCommand(const acommand: MarshaledAString): String;
// run a linux shell command and return output
// Adapted from http://chapmanworld.com/2017/04/06/calling-linux-commands-from-delphi/
var
handle: TStreamHandle;
data: array [0 .. 511] of uint8;
function bufferToString(buffer: pointer; maxSize: uint32): string;
var
cursor: ^uint8;
endOfBuffer: nativeuint;
begin
if not assigned(buffer) then
exit;
cursor := buffer;
endOfBuffer := nativeuint(cursor) + maxSize;
while (nativeuint(cursor) < endOfBuffer) and (cursor^ <> 0) do
begin
result := result + chr(cursor^);
cursor := pointer(succ(nativeuint(cursor)));
end;
end;
begin
result := '';
handle := popen(acommand, 'r');
try
while fgets(#data[0], sizeof(data), handle) <> nil do
begin
result := result + bufferToString(#data[0], sizeof(data));
end;
finally
pclose(handle);
end;
end;
function createQRCode(id, fn: string): string;
// Create qr-code using qrencode package
begin
deletefile(fn);
if fileExists(fn) then
raise Exception.create('Old file not deleted!');
// I am targeting rhel for now, so I know the path for sure
result := runCommand(MarshaledAString(UTF8STring('/usr/bin/qrencode -o ' + fn + ' ''' + id + '''')));
if not fileExists(fn) then
raise Exception.create('New file not created!');
end;
function testqr: String;
// Test QR Code creation with error handling
// QREncode does not output anything but who knows ;-)
begin
try
result := createQRCode('08154711', '/tmp/myqrcode.png');
except
on e: Exception do
begin
result := 'Error: ' + e.message;
end;
end;
end;
I wrote this code to do this task
uses
System.SysUtils,
System.Classes,
Posix.Base,
Posix.Fcntl;
type
TStreamHandle = pointer;
TLinuxUtils = class
public
class function RunCommandLine(ACommand : string) : TStringList;overload;
class function RunCommandLine(Acommand : string; Return : TProc<String>) : boolean; overload;
class function findParameter(AParameter : string) : boolean;
end;
function popen(const command: MarshaledAString; const _type: MarshaledAString): TStreamHandle; cdecl; external libc name _PU + 'popen';
function pclose(filehandle: TStreamHandle): int32; cdecl; external libc name _PU + 'pclose';
function fgets(buffer: pointer; size: int32; Stream: TStreamHAndle): pointer; cdecl; external libc name _PU + 'fgets';
implementation
class function TLinuxUtils.RunCommandLine(ACommand : string) : TStringList;
var
Handle: TStreamHandle;
Data: array[0..511] of uint8;
M : TMarshaller;
begin
Result := TStringList.Create;
try
Handle := popen(M.AsAnsi(PWideChar(ACommand)).ToPointer,'r');
try
while fgets(#data[0],Sizeof(Data),Handle)<>nil do begin
Result.Add(Copy(UTF8ToString(#Data[0]),1,UTF8ToString(#Data[0]).Length -1));//,sizeof(Data)));
end;
finally
pclose(Handle);
end;
except
on E: Exception do
Result.Add(E.ClassName + ': ' + E.Message);
end;
end;
class function TLinuxUtils.RunCommandLine(Acommand : string; Return : TProc<string>) : boolean;
var
Handle: TStreamHandle;
Data: array[0..511] of uint8;
M : TMarshaller;
begin
Result := false;
try
Handle := popen(M.AsAnsi(PWideChar(ACommand)).ToPointer,'r');
try
while fgets(#data[0],Sizeof(Data),Handle)<>nil do begin
Return(Copy(UTF8ToString(#Data[0]),1,UTF8ToString(#Data[0]).Length -1));//,sizeof(Data)));
end;
finally
pclose(Handle);
end;
except
on E: Exception do
Return(E.ClassName + ': ' + E.Message);
end;
end;
class function TLinuxUtils.findParameter(AParameter : string) : boolean;
var
I : Integer;
begin
Result := false;
for I := 0 to Pred(ParamCount) do
begin
Result := AParameter.ToUpper = ParamStr(i).ToUpper;
if Result then
Break;
end;
end;
You do not have to worry about MarshaledString.
The RunCommandLine function has 2 ways to be called. The first you have the return on a TStringList with all the lines that the console will return.
The second you can pass an anonymous method that will treat line by line of return of the command line.
I found this code on the net and it's working but I'm not sure if it's ok to directly read the variable in the main thread from another thread. In this example the flag (variable) is CancelCopy.
In general, I want to know how can I read the state of a variable from the main thread in another thread but immediately, without waiting.
type
TCopyEx = packed record
Source: String;
Dest: String;
Handle: THandle;
end;
PCopyEx = ^TCopyEx;
const
CFEX_CANCEL = WM_USER + 1;
var
CancelCopy:Boolean=False;
function CopyFileProgress(TotalFileSize, TotalBytesTransferred, StreamSize,
StreamBytesTransferred: LARGE_INTEGER; dwStreamNumber, dwCallbackReason: DWORD;
hSourceFile, hDestinationFile: THandle; lpData: Pointer): DWORD; stdcall;
begin
if CancelCopy then begin
SendMessage(THandle(lpData), CFEX_CANCEL, 0, 0);
result:=PROGRESS_CANCEL;
Exit;
end;
//rest of the code here.......
end;
function CopyExThread(p: PCopyEx):Integer;
var
Source: String;
Dest: String;
Handle: THandle;
Cancel: PBool;
begin
Source:=p.Source;
Dest:=p.Dest;
Handle:=p.Handle;
Cancel:=PBOOL(False);
CopyFileEx(PChar(Source), PChar(Dest), #CopyFileProgress, Pointer(Handle), Cancel, COPY_FILE_NO_BUFFERING);
Dispose(p);
result:=0;
end;
procedure TFormMain.ButtonCopyClick(Sender: TObject);
var
Params: PCopyEx;
ThreadID: Cardinal;
begin
cancelCopy := False;
New(Params);
Params.Source := EditOriginal.Text;
Params.Dest := EditCopied.Text;
Params.Handle := Handle;
CloseHandle(BeginThread(nil, 0, #CopyExThread, Params, 0, ThreadID));
end;
procedure TFormMain.ButtonCancelClick(Sender: TObject);
begin
cancelCopy := true;
end;
Technically, the code you have shown is fine, and will work as expected.
However, there is a small mistake in it. You are passing the wrong pointer value to the pbCancel parameter of CopyFileEx(). However, your code does not crash because the pointer you are passing is effectively being set to nil, and pbCancel will accept a nil pointer, thus CopyFileEx() will ignore the parameter.
What you are supposed to do is pass the address of a BOOL variable, which you can set to TRUE at any time to cancel the copy. CopyFileEx() will monitor that variable for you, you do not need to manually return PROGRESS_CANCEL from the callback when the variable is set (return PROGRESS_CANCEL if your callback encounters an error not related to the copy itself, and you want to abort the copy as a result of the error). I would not use a global variable for that, though. I would use a variable that is local to the Form that is performing the copy.
Try something more like this instead:
type
TFormMain = class(TForm)
...
private
CancelCopy: BOOL; // <-- BOOL, not Boolean
...
end;
...
type
TCopyEx = record
Source: String;
Dest: String;
Handle: HWND;
PCancelCopy: PBOOL;
end;
PCopyEx = ^TCopyEx;
const
CFEX_CANCEL = WM_USER + 1;
function CopyFileProgress(TotalFileSize, TotalBytesTransferred, StreamSize,
StreamBytesTransferred: LARGE_INTEGER; dwStreamNumber, dwCallbackReason: DWORD;
hSourceFile, hDestinationFile: THandle; lpData: Pointer): DWORD; stdcall;
begin
// no need to watch CancelCopy here...
// do normal status handling here as needed...
// use PCopyEx(lpData)^ as needed...
end;
function CopyExThread(p: PCopyEx): Integer;
begin
try
if not CopyFileEx(PChar(p.Source), PChar(p.Dest), #CopyFileProgress, p, p.PCancelCopy, COPY_FILE_NO_BUFFERING) then
begin
if GetLastError() = ERROR_REQUEST_ABORTED then
SendMessage(p.Handle, CFEX_CANCEL, 0, 0);
end;
finally
Dispose(p);
end;
Result := 0;
end;
procedure TFormMain.ButtonCopyClick(Sender: TObject);
var
Params: PCopyEx;
ThreadID: Cardinal;
begin
New(Params);
Params.Source := EditOriginal.Text;
Params.Dest := EditCopied.Text;
Params.Handle := Handle;
Params.PCancelCopy := #CancelCopy; // <-- pass address of CancelCopy here...
CancelCopy := FALSE;
CloseHandle(BeginThread(nil, 0, #CopyExThread, Params, 0, ThreadID));
end;
procedure TFormMain.ButtonCancelClick(Sender: TObject);
begin
CancelCopy := TRUE;
end;
With that said, something else to watch out for - you are passing the HWND from the TForm.Handle property to the thread. If the TForm ever destroys/recreates its HWND for any reason (and yes, it can happen) while the thread is still running, the TCopyEx.Handle value will be left pointing to an invalid window (or worse, to a new window that reuses the old HWND value).
In general, the TWinControl.Handle property is not thread-safe, and so for that reason alone, it is not a good idea to pass the HWND of a TWinControl object to a worker thread, unless you can guarantee the HWND will not be destroyed while the thread is running (and in this example, that is not guaranteed).
In this example, I would use a different HWND that is guaranteed to be persistent for the life of the thread, such as the TApplication.Handle window (messages sent to this window can be handled via TApplication.HookMainWindow()), or the result of calling AllocateHWnd().
For example:
type
TFormMain = class(TForm)
procedure FormDestroy(Sender: TObject);
...
private
CancelCopy: BOOL; // <-- BOOL, not Boolean
CopyFileExWnd: HWND;
procedure CopyFileExWndProc(var Message: TMessage);
...
end;
...
type
TCopyEx = record
Source: String;
Dest: String;
Handle: HWND;
PCancelCopy: PBOOL;
end;
PCopyEx = ^TCopyEx;
const
CFEX_CANCEL = WM_USER + 1;
function CopyFileProgress(TotalFileSize, TotalBytesTransferred, StreamSize,
StreamBytesTransferred: LARGE_INTEGER; dwStreamNumber, dwCallbackReason: DWORD;
hSourceFile, hDestinationFile: THandle; lpData: Pointer): DWORD; stdcall;
begin
...
end;
function CopyExThread(p: PCopyEx): Integer;
begin
try
if not CopyFileEx(
PChar(p.Source), PChar(p.Dest), #CopyFileProgress, p, p.PCancelCopy, COPY_FILE_NO_BUFFERING) then
begin
if GetLastError() = ERROR_REQUEST_ABORTED then
SendMessage(p.Handle, CFEX_CANCEL, 0, 0);
end;
finally
Dispose(p);
end;
Result := 0;
end;
procedure TFormMain.FormDestroy(Sender: TObject);
begin
if CopyFileExWnd <> 0 then
DeallocateHWnd(CopyFileExWnd);
end;
procedure TFormMain.ButtonCopyClick(Sender: TObject);
var
Params: PCopyEx;
ThreadID: Cardinal;
begin
if CopyFileExWnd = 0 then
CopyFileExWnd := AllocateHWnd(CopyFileExWndProc);
New(Params);
Params.Source := EditOriginal.Text;
Params.Dest := EditCopied.Text;
Params.Handle := CopyFileExWnd;
Params.PCancelCopy := #CancelCopy;
CancelCopy := FALSE;
CloseHandle(BeginThread(nil, 0, #CopyExThread, Params, 0, ThreadID));
end;
procedure TFormMain.ButtonCancelClick(Sender: TObject);
begin
CancelCopy := TRUE;
end;
procedure TFormMain.CopyFileExWndProc(var Message: TMessage);
begin
case Message.Msg of
CFEX_CANCEL: begin
...
end;
...
else
Message.Result := DefWindowProc(CopyFileExWnd, Message.Msg, Message.WParam, Message.LParam);
end;
end;
Me again, hopefully with more specific threading question.
I note that if I run the fine demos that Chris Rolliston provides here:
http://delphihaven.wordpress.com/2011/05/06/using-tmonitor-2/
with FastMM turned on to report memory leaks, that the threads themselves are leaked.
That's no problem for a small demo, but for my app, with tens of thousands of iterations of using the thread, it runs my humble 32-bit app out of memory. (I can't compile for 64 bit because I'm using the 32-bit only CrossTalk).
How do you ensure that the thread is freed when used with a threaded queue?
NEW CODE ADDED
program SimpleThreadQueueConsole;
{$APPTYPE CONSOLE}
{$R *.res}
uses
FastMM4,
System.SysUtils,
System.SyncObjs,
uPrimeThread in 'uPrimeThread.pas',
uPrimeThreadRunner in 'uPrimeThreadRunner.pas';
var
PR: TPrimeThreadRunner;
begin
Randomize;
ReportMemoryLeaksOnShutdown := True;
PR := TPrimeThreadRunner.Create;
try
PR.DoIt;
finally
PR.Free;
end;
end.
uPrimeThread.pas
unit uPrimeThread;
interface
uses
System.Classes
, Generics.Collections
;
type
TPrimeThread = class(TThread)
private
FOutQueue: TThreadedQueue<string>;
FInQueue: TThreadedQueue<string>;
function IsPrime(const NumberToCheck: integer): boolean;
public
constructor Create(aCreateSuspended: Boolean; aInQueue: TThreadedQueue<string>; aOutQueue: TThreadedQueue<string>);
procedure Execute; override;
end;
implementation
uses
System.SysUtils
, System.SyncObjs
;
const
MaxPrime = 999;
{ TPrimeThread }
constructor TPrimeThread.Create(aCreateSuspended: Boolean; aInQueue, aOutQueue: TThreadedQueue<string>);
begin
inherited Create(aCreateSuspended);
FOutQueue := aOutQueue;
FInQueue := aInQueue;
FreeOnTerminate := True;
end;
procedure TPrimeThread.Execute;
var
S: string;
ThreadID: TThreadID;
NumberToCheck: integer;
begin
ThreadID := TThread.CurrentThread.ThreadID;
FOutQueue.PushItem(Format('Thread %d started...', [ThreadID]));
while (FInQueue.PopItem(S) = wrSignaled) do
begin
NumberToCheck := Random(MaxPrime);
if IsPrime(NumberToCheck) then
begin
FOutQueue.PushItem(Format('%s using thread %d: %d is prime', [S, ThreadID, NumberToCheck]));
end else
begin
FOutQueue.PushItem(Format('%s using thread %d: %d is NOT prime', [S, ThreadID, NumberToCheck]));
end;
end;
end;
function TPrimeThread.IsPrime(const NumberToCheck: Integer): boolean;
// This is really bad on purpose to make the threads work a little harder
var
i: integer;
begin
Result := True;
if NumberToCheck in [0, 1] then
begin
Result := False;
Exit;
end;
for i := 2 to NumberToCheck - 1 do
begin
if NumberToCheck mod i = 0 then
begin
Result := False;
Exit;
end;
end;
end;
end.
uPrimeThreadRunner.pas
unit uPrimeThreadRunner;
interface
uses
System.SyncObjs
, Generics.Collections
, System.SysUtils
, uPrimeThread
;
const
ThreadCount = 4;
type
TPrimeThreadRunner = class
private
FTotalThreads: TCountdownEvent;
FInQueue, FOutQueue: TThreadedQueue<string>;
FCurrentEntry: integer;
procedure DrainTheQueue;
procedure AddEntry;
public
ThreadArray: array[1..ThreadCount] of TPrimeThread;
procedure DoIt;
end;
implementation
const
NumberOfEntries = 10;
procedure TPrimeThreadRunner.DrainTheQueue;
var
S: string;
begin
while FOutQueue.PopItem(S) = wrSignaled do
WriteLn(S);
end;
procedure TPrimeThreadRunner.AddEntry;
var
S: string;
begin
Inc(FCurrentEntry);
S := Format('Entry %d:', [FCurrentEntry]) ;
FInQueue.PushItem(S);
end;
procedure TPrimeThreadRunner.DoIt;
var
i: integer;
begin
FCurrentEntry := 0;
FTotalThreads := TCountdownEvent.Create(1);
FInQueue := TThreadedQueue<string>.Create(10, 1000, 1000);
FOutQueue := TThreadedQueue<string>.Create(10, 1000, 1000);
for i := 1 to ThreadCount do
begin
FTotalThreads.AddCount;
try
ThreadArray[i] := TPrimeThread.Create(True, FInQueue, FOutQueue);
ThreadArray[i].Start;
finally
FTotalThreads.Signal;
end;
end;
for I := 1 to NumberOfEntries do
begin
AddEntry;
end;
DrainTheQueue;
FTotalThreads.Signal;
FTotalThreads.WaitFor;
FTotalThreads.Free;
FInQueue.Free;
FOutQueue.Free;
Readln;
end;
end.
There is an error in your code, I can replicate it in XE3 and XE6.
The critical error is here:
for I := 1 to NumberOfEntries do
begin
AddEntry;
end;
DrainTheQueue; // <-- All threads may not be ready when this call finish
FTotalThreads.Signal; // This makes nothing
FTotalThreads.WaitFor; // This makes nothing
FTotalThreads.Free; // This makes nothing
FInQueue.Free; // You may now free a queue in operation
FOutQueue.Free; // You may now free a queue in operation
You can't rely on DrainTheQueue to collect all items until the threads are done.
When freeing the queues while one or more primethreads are running, the threads will operate on freed objects.
The easiest way to syncronize the finish of the threads, is to finish them with an empty string.
Call this before DrainTheQueue:
for i := 1 to ThreadCount do
FInQueue.PushItem('');
And change the PrimeThread.Execute like this:
procedure TPrimeThread.Execute;
var
S: string;
ThreadID: TThreadID;
NumberToCheck: integer;
begin
ThreadID := TThread.CurrentThread.ThreadID;
FOutQueue.PushItem(Format('Thread %d started...', [ThreadID]));
try
while NOT Terminated do
begin
if (FInQueue.PopItem(S) = wrSignaled) then
begin
if (S = '') then // Stop executing
Exit;
NumberToCheck := Random(MaxPrime);
if IsPrime(NumberToCheck) then
begin
FOutQueue.PushItem(Format('%s using thread %d: %d is prime', [S, ThreadID, NumberToCheck]));
end else
begin
FOutQueue.PushItem(Format('%s using thread %d: %d is NOT prime', [S, ThreadID, NumberToCheck]));
end;
end;
end;
finally
FOutQueue.PushItem(Format('Thread %d ended ...', [ThreadID]));
end;
end;
You should also pass the FTotalThreads object to the worker threads and let them do the signal when they finish executing.
DrainTheQueue must be called after FTotalThreads.Waitfor.