I have this automation test running on my local.
But every time when the test is executed, there is a onPrepare hook takes 3 to 8 minutes randomly.
Here is my config.js
exports.config = {
//
// ====================
// Runner Configuration
// ====================
//
// WebdriverIO allows it to run your tests in arbitrary locations (e.g. locally or
// on a remote machine).
runner: 'local',
//
// ==================
// Specify Test Files
// ==================
// Define which test specs should run. The pattern is relative to the directory
// from which `wdio` was called.
//
// The specs are defined as an array of spec files (optionally using wildcards
// that will be expanded). The test for each spec file will be run in a separate
// worker process. In order to have a group of spec files run in the same worker
// process simply enclose them in an array within the specs array.
//
// If you are calling `wdio` from an NPM script (see https://docs.npmjs.com/cli/run-script),
// then the current working directory is where your `package.json` resides, so `wdio`
// will be called from there.
//
headless: true,
specs: [
'./features/logintrainer.feature'
],
// Patterns to exclude.
exclude: [
// 'path/to/excluded/files'
],
//
// ============
// Capabilities
// ============
// Define your capabilities here. WebdriverIO can run multiple capabilities at the same
// time. Depending on the number of capabilities, WebdriverIO launches several test
// sessions. Within your capabilities you can overwrite the spec and exclude options in
// order to group specific specs to a specific capability.
//
// First, you can define how many instances should be started at the same time. Let's
// say you have 3 different capabilities (Chrome, Firefox, and Safari) and you have
// set maxInstances to 1; wdio will spawn 3 processes. Therefore, if you have 10 spec
// files and you set maxInstances to 10, all spec files will get tested at the same time
// and 30 processes will get spawned. The property handles how many capabilities
// from the same test should run tests.
//
maxInstances: 10,
//
// If you have trouble getting all important capabilities together, check out the
// Sauce Labs platform configurator - a great tool to configure your capabilities:
// https://docs.saucelabs.com/reference/platforms-configurator
//
capabilities: [{
// maxInstances can get overwritten per capability. So if you have an in-house Selenium
// grid with only 5 firefox instances available you can make sure that not more than
// 5 instances get started at a time.
maxInstances: 5,
//
browserName: 'chrome',
acceptInsecureCerts: true
// 'goog:chromeOptions': {
// // to run chrome headless the following flags are required
// // (see https://developers.google.com/web/updates/2017/04/headless-chrome)
// args: ['--headless', '--disable-gpu'],
// }
// If outputDir is provided WebdriverIO can capture driver session logs
// it is possible to configure which logTypes to include/exclude.
// excludeDriverLogs: ['*'], // pass '*' to exclude all driver session logs
// excludeDriverLogs: ['bugreport', 'server'],
}],
//
// ===================
// Test Configurations
// ===================
// Define all options that are relevant for the WebdriverIO instance here
//
// Level of logging verbosity: trace | debug | info | warn | error | silent
logLevel: 'info',
//
// Set specific log levels per logger
// loggers:
// - webdriver, webdriverio
// - #wdio/applitools-service, #wdio/browserstack-service, #wdio/devtools-service, #wdio/sauce-service
// - #wdio/mocha-framework, #wdio/jasmine-framework
// - #wdio/local-runner
// - #wdio/sumologic-reporter
// - #wdio/cli, #wdio/config, #wdio/utils
// Level of logging verbosity: trace | debug | info | warn | error | silent
// logLevels: {
// webdriver: 'info',
// '#wdio/applitools-service': 'info'
// },
//
// If you only want to run your tests until a specific amount of tests have failed use
// bail (default is 0 - don't bail, run all tests).
bail: 0,
//
// Set a base URL in order to shorten url command calls. If your `url` parameter starts
// with `/`, the base url gets prepended, not including the path portion of your baseUrl.
// If your `url` parameter starts without a scheme or `/` (like `some/path`), the base url
// gets prepended directly.
baseUrl: 'https://test.com',
//
// Default timeout for all waitFor* commands.
waitforTimeout: 10000,
//
// Default timeout in milliseconds for request
// if browser driver or grid doesn't send response
connectionRetryTimeout: 120000,
//
// Default request retries count
connectionRetryCount: 3,
//
// Test runner services
// Services take over a specific job you don't want to take care of. They enhance
// your test setup with almost no effort. Unlike plugins, they don't add new
// commands. Instead, they hook themselves up into the test process.
services: [
['selenium-standalone', { drivers: { firefox: 'latest', chrome: 'latest', chromiumedge: 'latest' } }]
],
// Framework you want to run your specs with.
// The following are supported: Mocha, Jasmine, and Cucumber
// see also: https://webdriver.io/docs/frameworks
//
// Make sure you have the wdio adapter package for the specific framework installed
// before running any tests.
framework: 'cucumber',
//
// The number of times to retry the entire specfile when it fails as a whole
// specFileRetries: 1,
//
// Delay in seconds between the spec file retry attempts
// specFileRetriesDelay: 0,
//
// Whether or not retried specfiles should be retried immediately or deferred to the end of the queue
// specFileRetriesDeferred: false,
//
// Test reporter for stdout.
// The only one supported by default is 'dot'
// see also: https://webdriver.io/docs/dot-reporter
reporters: ['spec'],
//
// If you are using Cucumber you need to specify the location of your step definitions.
cucumberOpts: {
// <string[]> (file/dir) require files before executing features
require: ['./stepdefinations/given.js', './stepdefinations/when.js', './stepdefinations/then.js'],
// <boolean> show full backtrace for errors
backtrace: false,
// <string[]> ("extension:module") require files with the given EXTENSION after requiring MODULE (repeatable)
requireModule: ['#babel/register'],
// <boolean> invoke formatters without executing steps
dryRun: false,
// <boolean> abort the run on first failure
failFast: false,
// <string[]> (type[:path]) specify the output format, optionally supply PATH to redirect formatter output (repeatable)
format: ['pretty'],
// <boolean> hide step definition snippets for pending steps
snippets: true,
// <boolean> hide source uris
source: true,
// <string[]> (name) specify the profile to use
profile: [],
// <boolean> fail if there are any undefined or pending steps
strict: false,
// <string> (expression) only execute the features or scenarios with tags matching the expression
tagExpression: '',
// <number> timeout for step definitions
timeout: 60000,
// <boolean> Enable this config to treat undefined definitions as warnings.
ignoreUndefinedDefinitions: false
},
reporters: [
'spec',
['allure', {
outputDir: './reports/allure-results',
disableWebdriverStepsReporting: true,
disableWebdriverScreenshotsReporting: true,
}],
['json', {
outputDir: './reports/json-results'
}],
['junit', {
outputDir: './reports/junit-results',
outputFileFormat: function(options) {
return `results-${options.cid}.${options.capabilities}.xml`
}
}],
],
//
// =====
// Hooks
// =====
// WebdriverIO provides several hooks you can use to interfere with the test process in order to enhance
// it and to build services around it. You can either apply a single function or an array of
// methods to it. If one of them returns with a promise, WebdriverIO will wait until that promise got
// resolved to continue.
/**
* Gets executed once before all workers get launched.
* #param {Object} config wdio configuration object
* #param {Array.<Object>} capabilities list of capabilities details
*/
onPrepare: function (config, capabilities) {
console.log('Test is preparing');
},
/**
* Gets executed before a worker process is spawned and can be used to initialise specific service
* for that worker as well as modify runtime environments in an async fashion.
* #param {String} cid capability id (e.g 0-0)
* #param {[type]} caps object containing capabilities for session that will be spawn in the worker
* #param {[type]} specs specs to be run in the worker process
* #param {[type]} args object that will be merged with the main configuration once worker is initialised
* #param {[type]} execArgv list of string arguments passed to the worker process
*/
// onWorkerStart: function (cid, caps, specs, args, execArgv) {
// },
/**
* Gets executed just before initialising the webdriver session and test framework. It allows you
* to manipulate configurations depending on the capability or spec.
* #param {Object} config wdio configuration object
* #param {Array.<Object>} capabilities list of capabilities details
* #param {Array.<String>} specs List of spec file paths that are to be run
*/
// beforeSession: function (config, capabilities, specs) {
// },
/**
* Gets executed before test execution begins. At this point you can access to all global
* variables like `browser`. It is the perfect place to define custom commands.
* #param {Array.<Object>} capabilities list of capabilities details
* #param {Array.<String>} specs List of spec file paths that are to be run
* #param {Object} browser instance of created browser/device session
*/
// before: function (capabilities, specs) {
// },
/**
* Runs before a WebdriverIO command gets executed.
* #param {String} commandName hook command name
* #param {Array} args arguments that command would receive
*/
// beforeCommand: function (commandName, args) {
// },
/**
* Runs before a Cucumber feature
*/
// beforeFeature: function (uri, feature) {
// },
/**
* Runs before a Cucumber scenario
*/
// beforeScenario: function (world) {
// },
/**
* Runs before a Cucumber step
*/
// beforeStep: function (step, context) {
// },
/**
* Runs after a Cucumber step
*/
// afterStep: function (step, context) {
// },
/**
* Runs after a Cucumber scenario
*/
// afterScenario: function (world) {
// },
/**
* Runs after a Cucumber feature
*/
// afterFeature: function (uri, feature) {
// },
/**
* Runs after a WebdriverIO command gets executed
* #param {String} commandName hook command name
* #param {Array} args arguments that command would receive
* #param {Number} result 0 - command success, 1 - command error
* #param {Object} error error object if any
*/
// afterCommand: function (commandName, args, result, error) {
// },
/**
* Gets executed after all tests are done. You still have access to all global variables from
* the test.
* #param {Number} result 0 - test pass, 1 - test fail
* #param {Array.<Object>} capabilities list of capabilities details
* #param {Array.<String>} specs List of spec file paths that ran
*/
// after: function (result, capabilities, specs) {
// },
/**
* Gets executed right after terminating the webdriver session.
* #param {Object} config wdio configuration object
* #param {Array.<Object>} capabilities list of capabilities details
* #param {Array.<String>} specs List of spec file paths that ran
*/
// afterSession: function (config, capabilities, specs) {
// },
/**
* Gets executed after all workers got shut down and the process is about to exit. An error
* thrown in the onComplete hook will result in the test run failing.
* #param {Object} exitCode 0 - success, 1 - fail
* #param {Object} config wdio configuration object
* #param {Array.<Object>} capabilities list of capabilities details
* #param {<Object>} results object containing test results
*/
// onComplete: function(exitCode, config, capabilities, results) {
// },
/**
* Gets executed when a refresh happens.
* #param {String} oldSessionId session ID of the old session
* #param {String} newSessionId session ID of the new session
*/
//onReload: function(oldSessionId, newSessionId) {
//}
}
Attaching the logs here too:
`2021-06-04T06:27:55.993Z DEBUG #wdio/config:utils: Couldn't find ts-node package, no TypeScript compiling
Execution of 1 spec files started at 2021-06-04T06:27:56.014Z
2021-06-04T06:27:56.086Z INFO #wdio/cli:launcher: Run onPrepare hook
Test is preparing
2021-06-04T06:30:48.355Z INFO #wdio/cli:launcher: Run onWorkerStart hook
2021-06-04T06:30:48.359Z INFO #wdio/local-runner: Start worker 0-0 with arg: run,./config/wdio.conf.js
[0-0] 2021-06-04T06:30:49.021Z INFO #wdio/local-runner: Run worker command: run
[0-0] RUNNING in chrome - D:\webdriverIO-automation\features\logintrainer.feature
[0-0] 2021-06-04T06:30:49.974Z INFO webdriver: Initiate new session using the WebDriver protocol
I tried to comment the on prepare hook out, but still there is a onPrepare hook being run.`
This really slow the test down, appreciate if any idea on this issue.
PS: I'm using the latest webdriver-IO, nodeJS and chrome driver
Thank you
services: [
['selenium-standalone', { drivers: { firefox: 'latest', chrome: 'latest', chromiumedge: 'latest' },skipSeleniumInstall:true }]
],
add skipSeleniumInstall:true else it will try to download the drivers each time
Related
I have node application in which I want to run tasks on daily basis. So I want to use node-cron to schedule tasks but not like this:
var cron = require('node-cron');
cron.schedule('* * * * *', function(){
console.log('running a task every minute');
});
I need some generic solution so that at one place I have just empty body for cron job that takes different functions from a different location. That means in total we have two files, one which has just cron job function handler and other have a list of cron jobs. From where we want to run these jobs we just need require and some basic calling.
Can anyone suggest me this solution?
I got my solution. Using node-cron-job module I have achieved what I wanted. Let me explain in detail:-
First i created new file which contains jobs, lets name it jobs.js :-
// jobs.js
exports.first_job = {
on:"* * * * * " //runs every minute
},
job: function () {
console.log("first_job");
},
spawn: true
}
exports.second_job = {
on: "*/2 * * * * *", //runs every 2 second
job: function () {
console.log("second_job");
},
spawn: false // If false, the job will not run in a separate process.
}
Here, we define two jobs name first_job and second_job. We can define as many required.
Finally, we need to call these jobs from any location by these simple steps:-
// main.js
var cronjob = require('node-cron-job');
cronjob.setJobsPath(__dirname + '/jobs.js'); // Absolute path to the jobs module.
cronjob.startJob('first_job');
cronjob.startJob('second_job');
We can call all jobs in a single call like this:-
cronjob.startAllJobs();
After I update pjsip from v2.6 to v2.7.1,my app will assertion failed with fuction pjsua_set_snd_dev().
According pjsip's ticket #1941:
To make outgoing call:
func provider(_ provider: CXProvider, perform action: CXStartCallAction) {
/* 1. We must not start call audio here, and can only do so
* once the audio session has been activated by the system
* after having its priority elevated. So, make sure that the sound
* device is closed at this point.
*/
/* 2. Provide your own implementation to configure
* the audio session here.
*/
configureAudioSession()
/* 3. Make call with pjsua_call_make_call().
* Then use pjsua's on_call_state() callback to report significant
* events in the call's lifecycle, by calling iOS API
* CXProvider.reportOutgoingCall(with: startedConnectingAt:) and
* CXProvider.reportOutgoingCall(with: ConnectedAt:)
*/
/* 4. If step (3) above returns PJ_SUCCESS, call action.fulfill(),
* otherwise call action.fail().
*/
}
To handle incoming call:
func provider(_ provider: CXProvider, perform action: CXAnswerCallAction) {
/* 1. We must not start call audio here, and can only do so
* once the audio session has been activated by the system
* after having its priority elevated. So, make sure that the sound
* device is closed at this point.
*/
/* 2. Provide your own implementation to configure
* the audio session here.
*/
configureAudioSession()
/* 3. Answer the call with pjsua_call_answer().
*/
/* 4. If step (3) above returns PJ_SUCCESS, call action.fulfill(),
* otherwise call action.fail().
*/
}
To start sound device:
func provider(_ provider: CXProvider, didActivate audioSession: AVAudioSession) {
/* Start call audio media, now that the audio session has been
* activated after having its priority boosted.
*
* Call pjsua API pjsua_set_snd_dev() here.
*/
}
When calling pjsua API pjsua_set_snd_dev(),there show:
Assertion failed: (param && id!=PJMEDIA_AUD_INVALID_DEV), function
pjmedia_aud_dev_default_param, file ../src/pjmedia/audiodev.c, line
487.
I found that v2.7.1, in pjsua_set_no_snd_dev(),there is
pjsua_var.cap_dev = PJSUA_SND_NO_DEV; pjsua_var.play_dev =
PJSUA_SND_NO_DEV;
but in v2.6 ,do not have this.
PJSUA_SND_NO_DEV and PJMEDIA_AUD_INVALID_DEV are same as -3.
Is this a bug,or I confuse anything?
Steve have you added
App provides Voice over IP services required background modes in Capabilities
Or you can add in to directly in Info.plist
I'm trying to help a friend setup a bot for Picarto.tv, we have the bot LINK and there's no default plugin there for repeating messages, so I tried to make a very crude (seriously, it's terrible. I'm not a developer.) plugin for it, and I tried to use SetInterval/SetTimeout, and both times when I use them it will put the message in chat once, at the set interval, then it will wait, then after the interval it will say the message twice, then three times and so on.
It looks like this:
Time 1:
Testing...
Time 2:
Testing...
Testing...
And so on. Here's the code, as I said, it's terribly made, don't bash me too hard for it.
var api;
function handleChatMsg(data) {
var recursive = function () {
api.Messages.send("Testing Bot Repeat...");
setTimeout(recursive,15000);
}
recursive();
}
module.exports = {
meta_inf: {
name: "Repeat Message",
version: "1.0.0",
description: "Repeats a message every 5 minutes. Message and interval can be changed.",
author: "ZX6R"
},
load: function (_api) {
api = _api;
},
start: function () {
api.Events.on("userMsg", handleChatMsg);
}
}
Can anybody help me figure out why it's incrementally saying the message more times?
EDIT: Issue fixed, new code is
var api;
// Function to call for the repeating
function handleChatMsg() {
// This sets the interval of 5 minutes, and calls the variable. Edit the numbers after the comma to change the interval. You MUST put it into milliseconds.
setInterval(function(){xyz()}, 15000);
// This sets the variable, edit the text in "api.Messages.send" to change what the bot repeats.
var xyz = function()
{
api.Messages.send("Testing...")
}
}
// defines some information about the plugin, and sets up stuff we need.
module.exports = {
meta_inf: {
name: "Repeat Message",
version: "1.1.1",
description: "Repeats a message every 5 minutes. Message and interval can be changed.",
author: "ZX6R"
},
load: function (_api) {
api = _api;
},
start: function () {
handleChatMsg();
}
}
// The MIT License (MIT)
// Copyright (c) 2016 RedFalconv2 - ZX6R - WalnutGaming
//Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Let's not complicate things and make it simpler.
setInterval(function(){xyz()}, 300000);
var xyz = function()
{
//Do what you want your function to do.
}
Here function xyz will be called every 300,000 milliseconds which means 5 minutes.
Take a look at Node Cron btw if you are going to use it regularly for an application.
Perhaps something like the following:
// define and export module
var plugin = {};
module.exports = plugin;
// plugin metadata
plugin.meta_inf = {
name: "Repeat Message",
version: "1.0.0",
description: "Repeats a message every 5 minutes. Message and interval can be changed.",
author: "ZX6R"
};
/**
* To be called on plugin load.
* #param {Object} api - api instance.
*/
plugin.load = function(api){
plugin._api = api;
};
/**
* Called on plugin start.
*/
plugin.start = function(){
if(!plugin._api){
// api instance should have been available
throw new Error("api not defined, is plugin load()'ed ?");
}
// each user message event (re)configures the repeating timer
plugin._api.Events.on("userMsg",function(){
plugin._configure(true, "Testing echo bot...",15000);
});
};
/**
* Configure the repeating timer
* #param {Boolean} enabled - true to enable timer, false to disable
* #param {String} message - message to repeat, required if enabled
* #param {Number} interval - milliseconds between repeats, required if enabled
*/
plugin._configure = function(enabled, message, interval){
if(plugin._timer){
// always clear any old timer
clearInterval(plugin._timer);
}
if(enabled){
if(!message || !interval){
// message and interval are required
throw new Error("message and interval are required.");
}
// set a repeating timer
plugin._timer = setInterval(function(){
plugin._api.Messages.send(message);
},interval);
}
};
Notes:
user message events apply a new setting on timer. You can have the bot repeat user message, or something custom.
Or, you can only configure the timer once.
var crypto = require('crypto');
var sha = crypto.createHash('sha512').update(String(s));
var result = sha.digest('hex');
That's my current code.
How do I do this async? I'm planning to do the sha512 100,000 times.
Node's crypto module does not provide asynchronous SHA512 hashing at present, and although the createHash() stream interface looks asynchronous it will also execute in the main thread and block the event loop.
There is an issue open for this: https://github.com/nodejs/node/issues/678
In the interim, you can use #ronomon/crypto-async to do SHA512 asynchronously and concurrently in the threadpool without blocking the event loop, for multi-core throughput.
If you can not find any better solutions, this trick may help you:
You can create a standalone SHA-512 generator application, which receives your String "s" as standard input, generates the hash, and writes it out to the standard output.
From within your app, you can exec it via the child_process module, and catch the response with an event handler. There is an other stackoverflow thread, which may come handy about child_process:
Is it possible to execute an external program from within node js?
This way you can encapsulate the sync function into an async context. :)
Node.js runs in a single thread, so if you want to do asynchronous processing, you have to either:
use a module that implements threading natively;
spawn multiple Node.js processes.
The method I present below uses the latter approach.
Node.js API provides a module called cluster that allows you to fork your process as you would do if you were programming in C.
My approach breaks the input data (the strings you want to hash) into chunks, where each chunk is passed to a child worker process. When the worker finishes work on its chunk, it signals the master process, passing the results back.
The master node keeps running while the workers do their job, so it can do any unrelated asynchronous work without being blocked. When all workers finish, the master is signaled and it is free to further process the final results.
To run my test, you can simply do:
node parhash
My tests ran on an Intel Core i5 4670 with 8 GB RAM DDR3.
For your need of 100'000 strings, 1 worker completed in 450 ms, while 10 workers took 350 ms.
In a test with a million strings, 1 worker did the job in 4.5 seconds, while 10 workers did in 3.5 seconds.
Here is the code:
parhash.js
var
crypto = require('crypto'),
cluster = require('cluster');
var
STRING_COUNT = 1000000,
STRINGS_PER_WORKER = 100000,
WORKER_COUNT = Math.ceil(STRING_COUNT / STRINGS_PER_WORKER),
chunks = [],
nextChunkId = 0,
results = [],
startTime,
pendingWorkers = WORKER_COUNT;
/**
* Generates strings partitioned in WORKER_COUNT chunks.
* Each of these chunks will later be passed to a child process to be parsed asynchronously.
*
* You should replace this with your working data.
*/
function generateDemoStringChunks() {
var
si, wi,
chunk;
for (wi = 0; wi < WORKER_COUNT; wi++) {
chunk = [];
for (si = STRINGS_PER_WORKER * wi; (si < STRINGS_PER_WORKER * (wi + 1)) && (si < STRING_COUNT); si++) {
chunk.push(si.toString());
}
chunks.push(chunk);
}
}
/**
* After all workers finish processing, this will be executed.
*
* Here you should do whatever you want to process the resulting hashes.
*/
function mergeResults() {
results.sort(function compare(a, b) {
return a.id - b.id;
});
console.info('Summary:');
results.forEach(function (result) {
console.info('\tChunk %d: %d hashes (here is the first hash: "%s")', result.id, result.data.length, result.data[0]);
});
}
/**
* This will be called on the master side every time a worker finishes working.
*
* #param {object} worker the Worker that finished
* #param {{id: number, data: [string]}} result the result
*/
function processWorkerResult(worker, result) {
console.info('Worker %d finished computing %d hashes.', worker.id, result.data.length);
results.push(result);
worker.kill();
if (--pendingWorkers == 0) {
console.info('Work is done. Whole process took %d seconds.', process.hrtime(startTime)[0]);
mergeResults();
}
}
/**
* Gets a chunk of data available for processing.
*
* #returns {{id: number, data: [string]}} the chunk to be passed to the worker
*/
function getNextAvailableChunk() {
var chunk = {
id: nextChunkId,
data: chunks[nextChunkId]
};
nextChunkId++;
return chunk;
}
/**
* The master node will send a chunk of data every time a worker node
* signals it's ready to work.
*/
function waitForWorkers() {
cluster.on('online', function (worker) {
console.info('Worker %d is online.', worker.id);
worker.on('message', processWorkerResult.bind(null, worker));
worker.send(getNextAvailableChunk());
});
}
/**
* Start workers.
*/
function spawnWorkers() {
var wi;
for (wi = 0; wi < WORKER_COUNT; wi++) {
cluster.fork();
}
}
/**
* The hash function.
*
* #param {string} s a string to be hashed
* #returns {string} the hash string
*/
function hashString(s) {
return crypto.createHash('sha512').update(s).digest('hex');
}
/**
* A worker will wait for the master to send a chunk of data and will
* start processing as soon as it arrives.
*/
function processChunk() {
cluster.worker.on('message', function(chunk) {
var
result = [];
console.info('Worker %d received chunk %d with a load of %d strings.', cluster.worker.id, chunk.id, chunk.data.length);
chunk.data.forEach(function processChunk(s) {
result.push(hashString(s));
});
cluster.worker.send({
id: chunk.id,
data: result
});
});
}
function main() {
if (cluster.isMaster) {
/*
The master node will instantiate all required workers
and then pass a chunk of data for each one.
It will then wait for all of them to finish so it can
merge the results.
*/
startTime = process.hrtime();
generateDemoStringChunks();
spawnWorkers();
waitForWorkers();
} else {
/*
A worker node will wait for a chunk to arrive and
then will start processing it. When finished, it will
send a message back to the master node with the
resulting hashes.
*/
console.info('Worker %d is starting.', cluster.worker.id);
processChunk();
}
}
main();
I can't tell how well it would perform if it were implemented using threads because I haven't tested it. You could try WebWorker Threads if you want to do a benchmark (note: I haven't tried the WebWorkers module yet and I don't guarantee it works - you are on your own here).
I am using kue for my job queue, and I'd like to know without using the GUI how many jobs are still left, how many have failed, etc. How can I retrieve this kind of information?
For example, after a few minutes of starting the processing of the job queue, I'd like to o update the status of all jobs that failed so far to 'inactive', in order to restart them.
The only related question I could find on stackoverflow was this, however, it deals with one job at a time, after it fires a certain event as it is being processed. My concern is different, as I am interested in retrieving all jobs in the database with a certain status.
The answer to this question mentions the function .complete of the kue library, which retrieves all the completed jobs in the database. Are there similar functions for other possible job statuses?
I found a solution by browsing the kue source code. The following code achieves what I need:
var redis = require ('redis'),
kue = require ('kue'),
redisClient = redis.createClient(6379, "127.0.0.1");
kue.redis.createClient = function () {
return redisClient;
};
kue.app.listen(3000);
kue.Job.rangeByType ('job', 'failed', 0, 10, 'asc', function (err, selectedJobs) {
selectedJobs.forEach(function (job) {
job.state('inactive').save();
});
});
For reference, here is the relevant kue source code:
/queue/job.js:123:
/**
* Get jobs of `type` and `state`, with the range `from`..`to`
* and invoke callback `fn(err, ids)`.
*
* #param {String} type
* #param {String} state
* #param {Number} from
* #param {Number} to
* #param {String} order
* #param {Function} fn
* #api public
*/
exports.rangeByType = function(type, state, from, to, order, fn){
redis.client().zrange('q:jobs:' + type + ':' + state, from, to, get(fn, order));
};
Kue source code indicating that:
type is the job type
from, to is the job ranges by index (for example, you can specify load jobs from index 0 to 10, 11 jobs in total.)
order specifies the order of fetched jobs. Default is asc. You can also sort it by desc
The following works, uses the pre-existing queue object and hence, no double Redis connection issue as mentioned by Japrescott in the comments of the accepted answer.
queue.cardByType("notifications", "complete", function( err, count ) {
console.log(count);
});
Feel free to replace with a valid state, the following is a list of valid states.
inactive
complete
active
failed
delayed