How to Use Multithreading script - multithreading

Question

I found this Multithreading script at http://www.get-blog.com/?p=189
Param($Command = $(Read-Host "Enter the script file"),
[Parameter(ValueFromPipeline=$true,ValueFromPipelineByPropertyName=$true)]$ObjectList,
$InputParam = $Null,
$MaxThreads = 20,
$SleepTimer = 200,
$MaxResultTime = 120,
[HashTable]$AddParam = #{},
[Array]$AddSwitch = #()
)
Begin{
$ISS = [system.management.automation.runspaces.initialsessionstate]::CreateDefault()
$RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxThreads, $ISS, $Host)
$RunspacePool.Open()
If ($(Get-Command | Select-Object Name) -match $Command){
$Code = $Null
}Else{
$OFS = "`r`n"
$Code = [ScriptBlock]::Create($(Get-Content $Command))
Remove-Variable OFS
}
$Jobs = #()
}
Process{
Write-Progress -Activity "Preloading threads" -Status "Starting Job $($jobs.count)"
ForEach ($Object in $ObjectList){
If ($Code -eq $Null){
$PowershellThread = [powershell]::Create().AddCommand($Command)
}Else{
$PowershellThread = [powershell]::Create().AddScript($Code)
}
If ($InputParam -ne $Null){
$PowershellThread.AddParameter($InputParam, $Object.ToString()) | out-null
}Else{
$PowershellThread.AddArgument($Object.ToString()) | out-null
}
ForEach($Key in $AddParam.Keys){
$PowershellThread.AddParameter($Key, $AddParam.$key) | out-null
}
ForEach($Switch in $AddSwitch){
$Switch
$PowershellThread.AddParameter($Switch) | out-null
}
$PowershellThread.RunspacePool = $RunspacePool
$Handle = $PowershellThread.BeginInvoke()
$Job = "" | Select-Object Handle, Thread, object
$Job.Handle = $Handle
$Job.Thread = $PowershellThread
$Job.Object = $Object.ToString()
$Jobs += $Job
}
}
End{
$ResultTimer = Get-Date
While (#($Jobs | Where-Object {$_.Handle -ne $Null}).count -gt 0) {
$Remaining = "$($($Jobs | Where-Object {$_.Handle.IsCompleted -eq $False}).object)"
If ($Remaining.Length -gt 60){
$Remaining = $Remaining.Substring(0,60) + "..."
}
Write-Progress `
-Activity "Waiting for Jobs - $($MaxThreads - $($RunspacePool.GetAvailableRunspaces())) of $MaxThreads threads running" `
-PercentComplete (($Jobs.count - $($($Jobs | Where-Object {$_.Handle.IsCompleted -eq $False}).count)) / $Jobs.Count * 100) `
-Status "$(#($($Jobs | Where-Object {$_.Handle.IsCompleted -eq $False})).count) remaining - $remaining"
ForEach ($Job in $($Jobs | Where-Object {$_.Handle.IsCompleted -eq $True})){
$Job.Thread.EndInvoke($Job.Handle)
$Job.Thread.Dispose()
$Job.Thread = $Null
$Job.Handle = $Null
$ResultTimer = Get-Date
}
If (($(Get-Date) - $ResultTimer).totalseconds -gt $MaxResultTime){
Write-Error "Child script appears to be frozen, try increasing MaxResultTime"
Exit
}
Start-Sleep -Milliseconds $SleepTimer
}
$RunspacePool.Close() | Out-Null
$RunspacePool.Dispose() | Out-Null
}
I'm confused as how to use it though. In particular how to use the $ObjectList variable. I would like to use this ping script that I wrote to ping a list of 100,000 machines... But in it's current form it pings about 100 machines a minute. So it would take over 16 hours to complete.
$Computers = Get-Content -Path C:\Temp\Comps.txt
foreach ($Computer in $Computers) {
if (test-Connection -ComputerName $Computer -Count 1) {
"$Computer is Pinging"
}
Else {
"$Computer is not Pinging"
}
}
Any pointers as to how to integrate the ping script with the multithreading script to yield faster results?
I'm assuming that I would remove the $Computers variable from the ping script, and instead use the $ObjectList variable in the multithreading script. But all of my attempts to do so fail.

Answer 1

You can ping hosts asynchronously with [System.Net.NetworkInformation.Ping], no need to go crazy with runspaces. Moreover it will ping the host once, so it will be way much faster than test-connection. Below is a sample for a batch of hosts (about 90). I guess it's not a good idea to load 100K at once, probably split it in smaller batches and do one by one.
$hosts = "www.facebook.com,www.twitter.com,www.youtu.be,www.google.com,www.youtube.com,www.instagram.com,www.linkedin.com,www.pinterest.com,www.wordpress.com,www.blogspot.com,www.apple.com,www.adobe.com,www.tumblr.com,www.amazon.com,www.vimeo.com,www.flickr.com,www.microsoft.com,www.yahoo.com,www.godaddy.com,www.qq.com,www.vk.com,www.reddit.com,www.baidu.com,www.nytimes.com,www.buydomains.com,www.wp.com,www.statcounter.com,www.jimdo.com,www.blogger.com,www.github.com,www.weebly.com,www.soundcloud.com,www.myspace.com,www.addthis.com,www.theguardian.com,www.cnn.com,www.stumbleupon.com,www.gravatar.com,www.digg.com,www.addtoany.com,www.creativecommons.org,www.paypal.com,www.yelp.com,www.imdb.com,www.huffingtonpost.com,www.feedburner.com,www.issuu.com,www.wixsite.com,www.wix.com,www.dropbox.com,www.forbes.com,www.amazonaws.com,www.washingtonpost.com,www.bluehost.com,www.etsy.com,www.go.com,www.msn.com,www.wsj.com,www.weibo.com,www.fc2.com,www.eventbrite.com,www.parallels.com,www.ebay.com,www.livejournal.com,www.reuters.com,www.taobao.com,www.typepad.com,www.bloomberg.com,www.elegantthemes.com,www.eepurl.com,www.usatoday.com,www.about.com,www.medium.com,www.macromedia.com,www.xing.com,www.bing.com,www.time.com,www.tripadvisor.com,www.aol.com,www.constantcontact.com,www.latimes.com,www.list-manage.com,www.webs.com,www.opera.com,www.live.com,www.bandcamp.com,www.bbc.com,www.businessinsider.com,www.dailymotion.com,www.cpanel.com,www.disqus.com,www.sina.com.cn,www.spotify.com,www.wired.com,www.googleusercontent.com"
$hosts = $hosts -split ","
$tasks = #{}
foreach ($h in $hosts) { $tasks[$h] = [System.Net.NetworkInformation.Ping]::new().SendPingAsync($h)}
Write-Host "Waiting for batch is completed" -NoNewline
while($false -in $tasks.Values.IsCompleted) {sleep -Milliseconds 300; Write-Host "." -NoNewline}
$result = foreach($h in $hosts) {
$r = $tasks[$h].Result
[PSCustomObject]#{
host = $h
address = $r.Address.IPAddressToString
status = if($r.Address.IPAddressToString){$r.Status}else{"Failed"}
time = $r.RoundtripTime
bytes = $r.Buffer.Count
ttl = $r.Options.Ttl
}
}
$result | Format-Table -AutoSize

Answer 2

This script, which I have adapted from the link and author in the synopsis, has good performance for me. I have modified it to take not only a range of ip addresses, but also an array of host names. Because of that, some of the variable names have poor names, but this is not production code so I have not invested in it beyond what I have below.
function Global:Ping-IPRange {
<#
.SYNOPSIS
Sends ICMP echo request packets to a range of IPv4 addresses between two given addresses.
.DESCRIPTION
This function lets you sends ICMP echo request packets ("pings") to
a range of IPv4 addresses using an asynchronous method.
Therefore this technique is very fast but comes with a warning.
Ping sweeping a large subnet or network with many swithes may result in
a peak of broadcast traffic.
Use the -Interval parameter to adjust the time between each ping request.
For example, an interval of 60 milliseconds is suitable for wireless networks.
The RawOutput parameter switches the output to an unformated
[System.Net.NetworkInformation.PingReply[]].
.INPUTS
None
You cannot pipe input to this funcion.
.OUTPUTS
The function only returns output from successful pings.
Type: System.Net.NetworkInformation.PingReply
The RawOutput parameter switches the output to an unformated
[System.Net.NetworkInformation.PingReply[]].
.NOTES
Author : G.A.F.F. Jakobs
Created : August 30, 2014
Version : 6
.EXAMPLE
Ping-IPRange -StartAddress 192.168.1.1 -EndAddress 192.168.1.254 -Interval 20
IPAddress Bytes Ttl ResponseTime
--------- ----- --- ------------
192.168.1.41 32 64 371
192.168.1.57 32 128 0
192.168.1.64 32 128 1
192.168.1.63 32 64 88
192.168.1.254 32 64 0
In this example all the ip addresses between 192.168.1.1 and 192.168.1.254 are pinged using
a 20 millisecond interval between each request.
All the addresses that reply the ping request are listed.
.EXAMPLE
Ping-IPRange -HostName "host1"
Ping-IPRange -HostName #('host1', 'host2')
Ping-IPRange -HostName #('www.microsoft.com', 'www.google.com')
.LINK
http://gallery.technet.microsoft.com/Fast-asynchronous-ping-IP-d0a5cf0e
#>
[CmdletBinding(ConfirmImpact='Low')]
Param(
[parameter(Mandatory = $true, Position = 0, ParameterSetName='range')]
[System.Net.IPAddress]$StartAddress,
[parameter(Mandatory = $true, Position = 1, ParameterSetName='range')]
[System.Net.IPAddress]$EndAddress,
[parameter(Mandatory = $true, Position = 0, ParameterSetName='list')]
[string[]]$HostName,
[int]$Interval = 30,
[Switch]$RawOutput = $false
)
$timeout = 2000
function New-Range ($start, $end) {
$addrList = [System.Collections.ArrayList]::new()
[byte[]]$BySt = $start.GetAddressBytes()
[Array]::Reverse($BySt)
[byte[]]$ByEn = $end.GetAddressBytes()
[Array]::Reverse($ByEn)
$i1 = [System.BitConverter]::ToUInt32($BySt,0)
$i2 = [System.BitConverter]::ToUInt32($ByEn,0)
for($x = $i1;$x -le $i2;$x++){
$ip = ([System.Net.IPAddress]$x).GetAddressBytes()
[Array]::Reverse($ip)
$null = $addrList.Add([System.Net.IPAddress]::Parse($($ip -join '.')))
}
,$addrList
}
if ($HostName)
{
$IPrange = #($HostName)
}
else
{
[System.Collections.ArrayList]$IPrange = New-Range $StartAddress $EndAddress
}
$IpTotal = $IPrange.Count
Get-Event -SourceIdentifier "ID-Ping*" | Remove-Event
Get-EventSubscriber -SourceIdentifier "ID-Ping*" | Unregister-Event
$index = 0
foreach ($ip in $IPrange){
if ($HostName)
{
[string]$VarName = "Ping_" + $ip + "_" + [guid]::NewGuid().ToString()
}
else
{
[string]$VarName = "Ping_" + $ip.Address
}
New-Variable -Name $VarName -Value (New-Object System.Net.NetworkInformation.Ping)
Register-ObjectEvent -InputObject (Get-Variable $VarName -ValueOnly) -EventName PingCompleted -SourceIdentifier "ID-$VarName"
(Get-Variable $VarName -ValueOnly).SendAsync($ip,$timeout,$VarName)
try{
$pending = (Get-Event -SourceIdentifier "ID-Ping*").Count
}catch [System.InvalidOperationException]{
Write-Verbose "Ping-IPrange : InvalidOperationException" -Verbose
}
finally{
Remove-Variable $VarName
}
#$index = [array]::indexof($IPrange,$ip)
if ($HostName)
{
Write-Progress -Activity "Sending ping to" -Id 1 -status $ip -PercentComplete (($index / $IpTotal) * 100)
}
else
{
Write-Progress -Activity "Sending ping to" -Id 1 -status $ip.IPAddressToString -PercentComplete (($index / $IpTotal) * 100)
}
Write-Progress -Activity "ICMP requests pending" -Id 2 -ParentId 1 -Status ($index - $pending) -PercentComplete (($index - $pending)/$IpTotal * 100)
Start-Sleep -Milliseconds $Interval
$index++
}
Write-Progress -Activity "Done sending ping requests" -Id 1 -Status 'Waiting' -PercentComplete 100
While($pending -lt $IpTotal){
Wait-Event -SourceIdentifier "ID-Ping*" | Out-Null
Start-Sleep -Milliseconds 10
$pending = (Get-Event -SourceIdentifier "ID-Ping*").Count
Write-Progress -Activity "ICMP requests pending" -Id 2 -ParentId 1 -Status ($IpTotal - $pending) -PercentComplete (($IpTotal - $pending)/$IpTotal * 100)
}
if($RawOutput){
$Reply = Get-Event -SourceIdentifier "ID-Ping*" | ForEach {
If($_.SourceEventArgs.Reply.Status -eq "Success"){
$_.SourceEventArgs.Reply
}
Unregister-Event $_.SourceIdentifier
Remove-Event $_.SourceIdentifier
}
}else{
$events = Get-Event -SourceIdentifier "ID-Ping*"
$Reply = [System.Collections.ArrayList]::new()
$i = 0
foreach ($event in $events) {
If($event.SourceEventArgs.Reply.Status -eq "Success"){
if ($HostName)
{
$null = $Reply.Add(
[PSCustomObject]#{
"HostName" = $HostName[$i]
"IPAddress" = $event.SourceEventArgs.Reply.Address
"Bytes" = $event.SourceEventArgs.Reply.Buffer.Length
"Ttl" = $event.SourceEventArgs.Reply.Options.Ttl
"ResponseTime" = $event.SourceEventArgs.Reply.RoundtripTime
}
)
}
else
{
$null = $Reply.Add(
[PSCustomObject]#{
"IPAddress" = $event.SourceEventArgs.Reply.Address
"Bytes" = $event.SourceEventArgs.Reply.Buffer.Length
"Ttl" = $event.SourceEventArgs.Reply.Options.Ttl
"ResponseTime" = $event.SourceEventArgs.Reply.RoundtripTime
}
)
}
}
else{
$addr = ($event.SourceIdentifier -split '_')[1]
$ip = ((([System.Net.IPAddress]$addr).IPAddressToString).Split('.'))
[Array]::Reverse($ip)
$ip = $ip -join '.'
if ($HostName)
{
$null = $Reply.Add(
[PSCustomObject]#{
"HostName" = $HostName[$i]
"IPAddress" = '0.0.0.0'
"Bytes" = -1
"Ttl" = -1
"ResponseTime" = -1
}
)
}
else
{
$null = $Reply.Add(
[PSCustomObject]#{
"IPAddress" = $ip
"Bytes" = -1
"Ttl" = -1
"ResponseTime" = -1
}
)
}
}
Unregister-Event $event.SourceIdentifier
Remove-Event $event.SourceIdentifier
$i++
}
}
if($Reply -eq $Null){
Write-Verbose "Ping-IPrange : No ip address responded" -Verbose
}
return ,$Reply
}