How to mock Tedious Module SQL Connection functions in JEST - jestjs

I am using Azure functions written in Nodejs.
I have logic to insert into DB after all actions are completed. This gets called from main index.js after some api calls. So, from test class im expecting to mock database methods. and cant understand mocking much!
Below is the code for Database logic.
'use strict';
const { Connection, Request, TYPES } = require('tedious');
const config = {
server: process.env.myDB_Server,
authentication: {
type: 'default',
options: {
userName: process.env.myDB_User,
password: process.env.myDB_Pwd
}
},
options: {
encrypt: true,
database: process.env.myDB_Name
}
};
const myDB = process.env.myDB;
module.exports = async(context, myPayload, last_Modified_By, status, errorCode, errorMsg, errorDescription) => {
try {
context.log('inside azureTable function');
let connection = new Connection(config);
connection.on('connect', function(err1) {
if (err1) {
context.log('Error connection.OnConnect to DB:::', err1.message);
//logger.error('Error connection.OnConnect to DB::', err1);
let dbStatus = {};
dbStatus["status"] = 400;
dbStatus["message"] = err1.message;
context.res.body["dbStatus"] = dbStatus;
context.done();
} else {
context.log('Database Connection Successful.');
var request = new Request("INSERT INTO " + myDB + " (Correlation_Id,Created_Date,LastModified_Date,Last_Modified_By,Status_CD,Error_Code,Error_Msg,Error_Description,Payload) VALUES (#correlationId,CURRENT_TIMESTAMP,CURRENT_TIMESTAMP,#Last_Modified_By,#Status_CD,#Error_Code,#Error_Msg,#Error_Description,#Payload);", function(err2) {
if (err2) {
context.log('Error inserting records to DB::', err2.message);
//logger.error('Error inserting records to DB::' + err2.message);
let dbStatus = {};
dbStatus["status"] = 400;
dbStatus["message"] = err2.message;
context.res.body["dbStatus"] = dbStatus;
context.done();
}
});
request.addParameter('correlationId', TYPES.NVarChar, JSON.parse(myPayload).correlationId);
request.addParameter('Last_Modified_By', TYPES.NVarChar, last_Modified_By);
request.addParameter('Status_CD', TYPES.NVarChar, status);
request.addParameter('Error_Code', TYPES.Int, errorCode);
request.addParameter('Error_Msg', TYPES.NVarChar, errorMsg);
request.addParameter('Error_Description', TYPES.NVarChar, errorDescription);
request.addParameter('Payload', TYPES.NVarChar, myPayload);
// Close the connection after the final event emitted by the request, after the callback passes
request.on("requestCompleted", function(rowCount, more) {
context.log('Records Successfully inserted into DB');
connection.close();
let dbStatus = {};
dbStatus["status"] = 201;
dbStatus["message"] = "Records Successfully inserted into DB";
context.res.body["dbStatus"] = dbStatus;
context.done();
});
connection.execSql(request);
}
});
connection.connect();
} catch (err) {
context.log('Error in main function::', err.message);
//logger.error('Error in main function::' + err.message);
let dbStatus = {};
dbStatus["status"] = 400;
dbStatus["message"] = err.message;
context.res.body["dbStatus"] = dbStatus;
context.done();
}
};
How can i mock the connection.on connect or request = new Request without actually hitting DB ?
I tried this, but its going to actual connection.
index.test.js
test('return 500 when db connection fails" ', async() => {
const tedious = require('tedious');
const connectionMock = jest.spyOn(tedious, 'connect');
connectionMock.mockImplementation(() => {
return {
}
});
//calling index js
}, 15000);
test('return 500 when db connection fails" ', async() => {
const tedious = require('tedious');
const connectionMock = jest.spyOn(tedious, 'Connection');
connectionMock.mockImplementation(() => {
{
throw new Error('some err');
}
});
//calling index js
}, 15000);
After going through some docs, tried below with no luck. Jest is not setting return value and gets timed out.
jest.mock('tedious', () => ({
Connection: jest.fn(() => ({
connect: jest.fn().mockReturnValue('err'),
on: jest.fn().mockReturnValue('err')
}))
}))
/* jest.mock('tedious', () => ({
Connection: jest.fn(() => ({
connect: jest.fn(() => (connect, cb) => cb(null)),
on: jest.fn(() => (connect, cb) => cb('err'))
}))
})) */

Finally I figured it out.
Issue was the mocking params not being set correctly. Unnecessarily used Jest.fn() for inner methods which actually doesn't help.
Here is the final solution:
jest.mock('tedious', () => ({
Connection: jest.fn(() => ({
connect: () => {},
on: (connect, cb) => cb(),
close: () => {},
execSql: () => {}
})),
TYPES: jest.fn(),
Request: jest.fn(() => ({
constructor: (sqlString, cb) => cb('err', null, null),
addParameter: (name, type, value) => {},
on: (requestCompleted, cb) => cb('rowCount', 'more')
}))
}))

Related

Jest Mockup ldap in nodejs

Here is my AD class code I am using ldapjs library and trying to mock its add method but facing some warning which cause code coverage issue
const ldap = require('ldapjs');
class AD {
/**
* Create Connection Active Directory
*/
async createConnection(){
return new Promise((resolve, reject) => {
var options = {
'rejectUnauthorized': false,
};
this.ADConnnection = ldap.createClient({
url: [
process.env.LDAP_SERVER_1 + ':' + process.env.LDAP_SERVER_PORT,
],
reconnect: true,
tlsOptions: options
});
this.ADConnnection.on('error', (err) => {
reject(err)
})
this.ADConnnection.bind(this.ldapUsername, this.ldapPassword, async (err) => {
if (err) {
reject(err)
}
});
resolve(true)
});
}
/**
* Create Record in Active Directory
* #param {*} oRequest
* #param {*} oEntry
*/
async create(oRequest, oADCreateUserDT) {
const sUsername = oRequest.vpnUsername;
this.adOu = oRequest.apiUser;
let oEntry = oADCreateUserDT.ADCreateUserData();
if(oEntry.hasOwnProperty('msRADIUSFramedIPAddress')){
this.adOu = ADConstant.AD_PARAMS.OU_DED;
oADCreateUserDT.setTitle(ADConstant.AD_PARAMS.OU_DED);
oEntry = oADCreateUserDT.ADCreateUserData();
}
return new Promise(async (resolve, reject) => {
this.ADConnnection.add('cn=' + sUsername + ',ou=' + this.adOu + ',' + this.dc, oEntry, (err) => {
if (err) {
reject(err);
} else {
resolve(true);
}
});
await this.closeConnection()
});
}
async closeConnection() {
this.ADConnnection.unbind(err => {
if (err) {
reject(err)
}
}, () => {
this.ADConnnection.destroy();
});
}
}
module.exports = AD;
Now this is my test class (I am using jest for nodejs testing)
const AD = require("../../app/libraries/AD");
const ldap = require('ldapjs');
jest.mock('ldapjs', () => {
return jest.fn().mockImplementation(() => {
return {
createClient: jest.fn(),
add: jest.fn(() => Promise.resolve(true)),
}
})
});
const isDedicatedIP = true;
const oRequest = { uuid: "vpn00s01" }
const oEntry = { UserAccountControl: ADConstant.AD_PARAMS.AD_ACC_CONTROL_ENABLE }
const oSearch = { attributes: ["cn", "Accountexpires", "UserAccountControl"], scope: "sub", filter: "" }
describe('should test all cases of ldap function', () => {
test('should test constructor', async () => {
const oAD = new AD()
const response = oAD
expect(JSON.stringify(response)).toBe(JSON.stringify({ "dc": "dc=undefined,dc=undefined", "adOu": "purevpn", "objectclass": "user" }));
});
test('should set Adu', async () => {
const oAD = new AD()
const response = oAD.setAdOu(isDedicatedIP)
expect(JSON.stringify(response)).toBe(JSON.stringify({}));
});
test('should test create form ldap', async () => {
const oAD = new AD()
const response = oAD.create(oRequest, oADCreateUserDT)
expect(JSON.stringify(response)).toBe(JSON.stringify({}));
});
});
While running this jest test facing this issue
I don't understand how to mock my ldapjs methods. Even after adding in in mock implementation still having the same issue

Testing a NodeJS function that connects to SQL Server through Tedious with Jest, throws error: Cannot redefine property: Connection

Trying to test a Tedious connector function with Jest.
In code-file db_server.js I have this function below, it works as expected.
const AWS = require('aws-sdk')
const tedious = require('tedious')
const Request = require('tedious').Request;
const Connection = require('tedious').Connection;
exports.execSQLCmd = async function (conn_store, cmd) {
let config = await exports.getParameter(conn_store)
let conn_cfg = JSON.parse(config)
//connect
const connection = new Connection(conn_cfg);
connection.on('connect', function (err) {
if (err) {
console.log('CONNECTION ERROR: ' + err);
}
console.log('CONNECTED TO: ' + conn_cfg.server);
executeStatement();
});
connection.connect();
//execute the sql command
let result = '';
function executeStatement() {
let request = new Request(cmd, function (err) {
if (err) {
console.log('REQUEST ERROR: ' + err);
}
});
//add three event hooks to the request
request.on('row', function (columns) {
columns.forEach(function (column) {
result += (column.value === null ? '' : column.value) + ',';
});
result = result.replace(/,$/, '\r\n')
});
request.on('done', function (rowCount, more) {
console.log('SQL COMMAND COMPLETED: ' + rowCount + ' rows returned');
});
request.on('requestCompleted', function (rowCount, more) {
connection.close();
console.log('CONNECTION TO SERVER CLOSED');
});
connection.execSql(request);
}
return result;
}
exports.Request = Request
exports.Connection = Connection
exports.tedious = tedious
In the test file index.spec.js, I have the following test:
describe('Test execSQLCmd', () => {
const server = require("../src/db_server.js")
afterEach(() => {
jest.restoreAllMocks();
});
test('should query sql', async () => {
server.getParameter = jest.fn().mockReturnValue('{"xxx":123,"yyy":"hello"}')
server.execSQLCmd = jest.fn().mockReturnThis()
const TedConnect = jest.spyOn(server.tedious, 'Connection').mockReturnValue(jest.fn(() => ({
connect: () => {},
on: (connect, cb) => cb(),
close: () => {},
execSql: () => {}
})));
const TedRequest = jest.spyOn(server.tedious, 'Request').mockReturnValue(jest.fn(() => ({
constructor: (sqlString, cb) => cb('err', null, null),
on: [
(row, cb)=> cb('columns'),
(done, cb)=> cb('rowCount', 'more'),
(requestCompleted, cb) => cb('rowCount', 'more')
]
})));
let exec = server.execSQLCmd("/pstore/name", "select 1")
expect(TedConnect).toBeCalledTimes(1)
expect(TedRequest).toBeCalledTimes(1)
})
})
When I try to run the test, I get this error:
Test execSQLCmd › should query sql
TypeError: Cannot redefine property: Connection
at Function.defineProperty (<anonymous>)
298 | server.getParameter = jest.fn().mockReturnValue('{"xxx":123,"yyy":"hello"}')
299 | server.execSQLCmd = jest.fn().mockReturnThis()
> 300 | const TedConnect = jest.spyOn(server.tedious, 'Connection').mockReturnValue(jest.fn(() => ({
| ^
301 | connect: () => {},
302 | on: (connect, cb) => cb(),
303 | close: () => {},
at ModuleMockerClass.spyOn (node_modules/jest-mock/build/index.js:826:16)
at Object.<anonymous> (__tests__/index.spec.js:300:29)
Test Suites: 1 failed, 1 total
"tedious": "^14.1.0"
"jest": "^26.6.3",
I'm not seeing why this test fails.
Any help appreciated, thanks!
EDIT
Above error not happening after the fiollowing Test changes:
test('should query sql', async () => {
server.getParameter = jest.fn().mockReturnValue('{"xxx":123,"yyy":"hello"}')
server.execSQLCmd = jest.fn().mockReturnThis()
server.Connection = jest.fn().mockReturnValue(jest.fn(() => ({
connect: () => {},
on: (connect, cb) => cb(),
close: () => {},
execSql: () => {}
})))
server.Request = jest.fn().mockReturnValue(jest.fn(() => ({
constructor: (sqlString, cb) => cb('err', null, null),
//addParameter: (name, type, value) => {},
on: [
(row, cb)=> cb('columns'),
(done, cb)=> cb('rowCount', 'more'),
(requestCompleted, cb) => cb('rowCount', 'more')
]
})))
let exec = server.execSQLCmd("/pstore/name", "select 1")
expect(server.Connection).toBeCalledTimes(1)
expect(server.Request).toBeCalledTimes(1)
})
However, now server.execSQLCmd does't run, I cannot step into it with debug, exists right away and returns a large mock object.
Anyone see what I'm doing wrong here?
Thanks!
exports.execSQLCmd = async function (conn_store, cmd) {
let config = await exports.getParameter(conn_store)
let conn_cfg = JSON.parse(config)
//connect
exports.connect_exec(conn_cfg, cmd);
return result;
}
function connect_exec(conn_cfg, cmd) {
const connection = exports.get_connection(conn_cfg);
exports.on_connect(connection, cmd);
connection.connect();
return connection;
}
function on_connect(connection, cmd) {
connection.on('connect', function (err) {
if (err) {
console.log('CONNECTION ERROR: ' + err);
}
console.log('CONNECTED');
exports.execute_statement(connection, cmd);
});
}
function get_connection(conn_cfg) {
return new Connection(conn_cfg);
}
function execute_statement(connection, cmd) {
let request = exports.get_request(connection, cmd);
connection.execSql(request);
}
function get_request(connection, cmd) {
let request = new Request(cmd, function (err) {
if (err) {
console.log('REQUEST ERROR: ' + err);
}
});
//add three event hooks to the request
request.on('row', function (columns) {
columns.forEach(function (column) {
result += (column.value === null ? '' : column.value) + ',';
});
result = result.replace(/,$/, '\r\n');
});
request.on('done', function (rowCount, more) {
console.log('SQL COMMAND COMPLETED: ' + rowCount + ' rows returned');
});
request.on('requestCompleted', function (rowCount, more) {
connection.close();
console.log('CONNECTION TO SERVER CLOSED');
});
return request;
}
exports.AWS = AWS;
exports.connect_exec = connect_exec
exports.execute_statement = execute_statement
exports.get_request = get_request
exports.get_connection = get_connection
exports.on_connect = on_connect
Here's my test:
describe('Test execSQLCmd', async () => {
const server = require("../src/opportunity/db_server.js")
afterEach(() => {
jest.restoreAllMocks();
});
test('should query sql', async () => {
server.getParameter = jest.fn().mockReturnValue('{"xxx":123,"yyy":"hello"}')
server.get_connection = jest.fn().mockReturnValue({
connect: () => {},
on: (connect, cb) => cb(),
close: () => {} ,
execSql: () => {}
})
server.get_request = jest.fn().mockReturnValue({
constructor: (sqlString, cb) => cb('err', null, null),
on: [
(row, cb)=> cb('columns'),
(done, cb)=> cb('rowCount', 'more'),
(requestCompleted, cb) => cb('rowCount', 'more')
]
})
await server.execSQLCmd("/pstore/name", "select 1")
expect(server.get_connection).toHaveBeenCalled()
expect(server.get_request).toHaveBeenCalled()
})
})
Took me a day to write all the code for main functionality, ore than there is here.
Took me 2.5 days to figure out just this test. Silly!

Nodejs Streams - Help find my memory leak

So I have a process that selects from a table. I partition my select programmatically into 20 sub-selects. I then go through each on of those select and stream its data to an indexing client (solr). Every select memory jumps up and holds until I get an OOM.
I logged when each query went off and can be seen in in the following charts:
These correlate with each jump in the this memory graph:
14 of 20 queries ran before I oomed.
I see this behavior with code that is similar but with a delta that runs every 15 mins. Every delta holds some sort of memory until it eventually causes the server to crash with OOM (which recovers)
I have tried to track down issues with the delta past but gave up and just created a way to gracefully restart. What am I missing here?
Here is my entire code chain that makes this work... I know its a lot to look through but I figured as much detail as possible would help.
Library Stack:
"node": "~11.10.1"
"knex": "^0.20.9",
"oracledb": "^4.0.0"
"camelize2": "^1.0.0"
Knex - DB connection factory
'use strict'
const objection = require('objection')
const knex = require('knex')
module.exports = function ObjectionFactory(log) {
class MyObjection extends objection.Model {
constructor() {
super()
}
static get tableName() {
return ''
}
}
MyObjection.pickJsonSchemaProperties = true
log.info('Connecting to Oracle Pluggable...', {
host: 'myHost',
username: 'myUser',
database: 'myDatabase"
})
const knexInstance = knex({
client: 'oracledb',
connection: 'connectionInfo',
pool: {
min: 0,
max: 10
},
acquireConnectionTimeout: 10000
})
process.once('SIGINT', () => {
log.info('Disconnecting from Oracle Pluggable.')
knexInstance.destroy()
.then(() => process.exit(0))
.catch(() => process.exit(1))
})
// Shut down cleanly for nodemon
process.once('SIGUSR2', () => {
log.info('Disconnecting from Oracle Pluggable')
knexInstance.destroy()
.then(() => process.kill(process.pid, 'SIGUSR2'))
.catch(() => process.kill(process.pid, 'SIGUSR2'))
})
const knexBoundClass = MyObjection.bindKnex(knexInstance)
knexBoundClass.tag = 'Oracle Connection'
return knexBoundClass
}
My Select Stream Code:
module.exports = function oracleStream(log, MyObjection) {
const knex = MyObjection.knex()
const fetchArraySize = 10000
const outFormat = oracledb.OBJECT
return {
selectStream
}
async function selectStream(sql, bindings = [], fetchSize = fetchArraySize) {
let connection = await knex.client.acquireConnection()
log.info(`Fetch size is set to ${fetchSize}`)
let select = connection.queryStream(sql, bindings, {
fetchArraySize: fetchSize,
outFormat: outFormat
})
select.on('error', (err) => {
log.error('Oracle Error Event', err)
knex.client.releaseConnection(connection)
})
select.on('end', () => {
log.info('Destroying the Stream')
select.destroy()
})
select.on('close', () => {
log.info('Oracle Close Event')
knex.client.releaseConnection(connection)
select = null
connection = null
})
return select
}
}
My index/stream pipeline code
async function indexJob() {
const reindexStartTime = new moment().local()
let rowCount = 0
log.info('Reindex Started at', reindexStartTime.format())
let queryNumber = 1
const partitionedQueries = ['Select * from table where 1=1', 'Select * from table where 2=2', 'Select * from table where 3=3'] //There would be 20 queries in this array
let partitionedQueriesLength = partitionedQueries.length
while (partitionedQueries.length > 0) {
let query = partitionedQueries.pop()
log.info('RUNNING Query', {
queryNumber: `${queryNumber++} of ${partitionedQueriesLength}`,
query: query
})
let databaseStream = await oracleStream.selectStream(query, [], 10000) //10k represents the oracle fetch size
databaseStream.on('data', () => {
rowCount++
})
let logEveryFiveSec = setInterval(() => {
log.info('Status: ', getReindexInfo(reindexStartTime, rowCount))
}, 5000)
try {
let pipeline = util.promisify(stream.pipeline)
await pipeline(
databaseStream,
camelizeAndStringify(),
streamReindex(core)
)
} catch (err) {
databaseStream.destroy(err)
throw new JobFailedError(err)
} finally {
databaseStream.destroy()
clearInterval(logEveryFiveSec)
}
}
}
function camelizeAndStringify() {
let first = true
const serialize = new Transform({
objectMode: true,
highWaterMark: 1000,
transform(chunk, encoding, callback) {
if (first) {
this.push('[' + JSON.stringify(camelize(chunk)))
first = false
} else {
this.push(',' + JSON.stringify(camelize(chunk)))
}
callback()
chunk = null
},
flush(callback) {
this.push(']')
callback()
}
})
return serialize
}
function streamReindex(core) {
const updateUrl = baseUrl + core + '/update'
const options = {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
'auth': `${user.username}:${user.password}`,
}
let postStream = https.request(updateUrl, options, (res) => {
let response = {
status: {
code: res.statusCode,
message: res.statusMessage
},
headers: res.headers,
}
if (res.statusCode !== 200) {
postStream.destroy(new Error(JSON.stringify(response)))
}
})
postStream.on('error', (err)=>{
throw new Error(err)
})
postStream.on('socket', (socket) => {
socket.setKeepAlive(true, 110000)
})
return postStream
}
EDIT 1:
I tried removing knex out of the equation by doing a singular connection to my db with the oracle library. Unfortunately I still see the same behavior.
This is how I changed my select to not use knex
async function selectStream(sql, bindings = [], fetchSize = fetchArraySize) {
const connectionInfo = {
user: info.user,
password: info.password,
connectString: info.host +'/'+info.database
}
const connection = await oracledb.getConnection(connectionInfo)
log.info('Connection was successful!')
log.info(`Fetch size is set to ${fetchSize}`)
let select = connection.queryStream(sql, bindings, {
fetchArraySize: fetchSize,
outFormat: outFormat
})
select.on('error', async (err) => {
log.error('Oracle Error Event', err)
await connection.close()
})
select.on('end', () => {
log.info('Destroying the Stream')
select.destroy()
})
select.on('close', async () => {
log.info('Oracle Close Event')
await connection.close()
})
return select
}

Jest mocking google-cloud/storage typescript

I have been trying to mock the #google-cloud/storage for my implementation so that I could test it without having to hit the cloud-storge in gcp and so far it has all been in vain
I have tried to mock the node_module scope folder using the jest doc and that didnt work out
Hence I tried using below
This is my implementation class
import { GcloudAuthenticationInstance } from '../common/services/gcloud.authentication';
import * as fs from 'fs';
import pump from 'pump';
import pino from 'pino';
import * as _ from 'lodash';
import {
ENV_NAME_DEV,
GCLOUD_DATABASE_BUCKET_DEV,
GCLOUD_DATABASE_BUCKET_PROD,
GCLOUD_ENV_STR_BUCKET_NAME,
GCLOUD_STORED_FILE_NAME_DEV,
GCLOUD_STORED_FILE_NAME_PROD,
GCLOUD_UPLOAD_FILE_DEV_LOCAL_PATH,
GCLOUD_UPLOAD_FILE_PROD_LOCAL_PATH,
} from '../common/util/app.constants';
import { PinoLoggerServiceInstance } from '../common/services/pino.logger.service';
import { AppUtilServiceInstance } from '../common/services/app.util.service';
export const uploadEnvFiles = async (env_name: string) => {
const LOGGER: pino.Logger = PinoLoggerServiceInstance.getLogger(__filename);
return new Promise(async (res, rej) => {
// This just returns the Storage() instance with keyFileName and projectID
//of google cloud console being set so authentication takes place
const str = GcloudAuthenticationInstance.createGcloudAuthenticationBucket();
const bucketToUpload = GCLOUD_ENV_STR_BUCKET_NAME;
let uploadLocalFilePath;
let destinationBucketPath;
if (!AppUtilServiceInstance.isNullOrUndefined(env_name)) {
uploadLocalFilePath = ENV_NAME_DEV === env_name ? GCLOUD_UPLOAD_FILE_DEV_LOCAL_PATH : GCLOUD_UPLOAD_FILE_PROD_LOCAL_PATH;
destinationBucketPath = ENV_NAME_DEV === env_name ? GCLOUD_DATABASE_BUCKET_DEV : GCLOUD_DATABASE_BUCKET_PROD;
}
LOGGER.info('after authentication');
pump(
fs.createReadStream(uploadLocalFilePath),
str
.bucket(bucketToUpload)
.file(destinationBucketPath)
.createWriteStream({
gzip: true,
public: true,
resumable: true,
})
)
.on('error', (err) => {
LOGGER.error('Error occured in uploading:', err);
rej({ status: 'Error', error: err, code: 500 });
})
.on('finish', () => {
LOGGER.info('Successfully uploaded the file');
res({ status: 'Success', code: 201, error: null });
});
});
};
export const downloadEnvFiles = async (env_name): Promise<any> => {
const LOGGER: pino.Logger = PinoLoggerServiceInstance.getLogger(__filename);
return new Promise(async (res, rej) => {
const str = GcloudAuthenticationInstance.createGcloudAuthenticationBucket();
try {
const [files] = await str.bucket(GCLOUD_ENV_STR_BUCKET_NAME).getFiles();
const filteredFile =
ENV_NAME_DEV === env_name
? _.find(files, (file) => {
c
return file.name.includes(GCLOUD_STORED_FILE_NAME_DEV);
})
: _.find(files, (file) => {
return file.name.includes(GCLOUD_STORED_FILE_NAME_PROD);
});
res({
status: 'Success',
code: 200,
error: null,
stream: str
.bucket(GCLOUD_ENV_STR_BUCKET_NAME)
.file(filteredFile.name)
.createReadStream()
});
} catch (err) {
LOGGER.error('Error in retrieving files from gcloud:'+err);
rej({ status: 'Error', error: err, code: 500 });
}
});
};
This is my jest ts
bucket.operations.int.spec.ts
I've tried to include the mock inline
import { GcloudAuthenticationInstance } from '../common/services/gcloud.authentication';
const { Storage } = require('#google-cloud/storage');
const { Bucket } = require('#google-cloud/storage');
import { File } from '#google-cloud/storage';
import { mocked } from 'ts-jest/utils'
const fs = require('fs');
import * as path from 'path';
import pump from 'pump';
import * as BucketOperations from './bucket.operations';
import { GCLOUD_ENV_STR_BUCKET_NAME } from '../common/util/app.constants';
const { PassThrough } = require('stream');
const fsMock = jest.mock('fs');
// Here we are trying to mock pump with a function returned
// since pump () is the actual fucntion, we are mocking the function to return a value
// which is just a value of "on" eventlistener.. so we indicate that this will be substituted
// with another mocked function
jest.genMockFromModule('#google-cloud/storage');
jest.mock('#google-cloud/storage', () => {
const mockedFile = jest.fn().mockImplementation(() => {
return {
File: jest.fn().mockImplementation(() => {
return {
name: 'dev.txt',
createReadStream: jest
.fn()
.mockReturnValue(
fs.createReadStream(
path.resolve(process.cwd(), './tests/cloud-storage/sample-read.txt')
)
),
createWriteStream: jest
.fn()
.mockReturnValue(
fs.createWriteStream(
path.resolve(process.cwd(), './tests/cloud-storage/sample-write.txt')
)
)
};
})
};
});
const mockedBUcket = jest.fn().mockImplementation(() => {
return {
Bucket: jest.fn().mockImplementation(() => {
return {
constructor: jest.fn().mockReturnValue('test-bucket'),
getFiles: jest.fn().mockReturnValue([mockedFile])
}
})
}
});
return {
Storage: jest.fn().mockImplementation(() => {
return {
constructor: jest.fn().mockReturnValue('test-storage'),
bucket: mockedBUcket,
file: mockedFile,
createWriteStream: jest.fn().mockImplementation(() =>
fs.createWriteStream(path.resolve(process.cwd(), './tests/cloud-storage/sample-write.txt')))
};
})
};
});
jest.mock('pump', () => {
const mPump = { on: jest.fn() };
return jest.fn(() => mPump);
});
describe('Test suite for testing bucket operations', () => {
const mockedStorage = mocked(Storage, true);
const mockeddFile = mocked(File, true);
const mockeddBucket = mocked(Bucket, true);
function cancelCloudStorageMock() {
//mockCloudStorage.unmock('#google-cloud/storage');
mockedStorage.mockClear();
mockeddBucket.mockClear();
mockeddFile.mockClear();
jest.unmock('#google-cloud/storage');
jest.requireActual('#google-cloud/storage');
}
function cancelFsMock() {
jest.unmock('fs');
jest.requireActual('fs');
}
afterEach(() => {
jest.clearAllMocks();
//jest.restoreAllMocks();
});
test('test for uploadfiles - success', async (done) => {
cancelFsMock();
pump().on = jest.fn(function(this: any, event, callback) {
if (event === 'finish') {
callback();
}
return this;
});
const actual = await BucketOperations.uploadEnvFiles('dev');
expect(actual).toEqual(
expect.objectContaining({
status: 'Success',
code: 201,
})
);
done();
});
test('test downloadEnvFiles - success', async (done) => {
jest.unmock('fs');
const fMock = (File.prototype.constructor = jest.fn().mockImplementation(() => {
return {
storage: new Storage(),
bucket: 'testBucket',
acl: 'test-acl',
name: 'dev.txt',
parent: 'parent bucket',
};
}));
const bucketGetFilMock = (Bucket.prototype.getFiles = jest.fn().mockImplementation(() => {
return [fMock];
}));
// Get files should be an array of File from google-cloud-storage
//Bucket.prototype.getFiles = jest.fn().mockReturnValue([mockedFsConstructor]);
//Storage.prototype.bucket = jest.fn().mockReturnValue(new Storage());
const mockReadable = new PassThrough();
const mockWritable = new PassThrough();
jest.spyOn(fs, 'createReadStream').mockReturnValue(
fs.createWriteStream(path.resolve(process.cwd(), './tests/cloud-storage/sample-read.txt'))
);
await BucketOperations.downloadEnvFiles('dev');
done();
});
});
This is the exception I end up with. Upon debugging I see that the mocked instances are trying to execute, but it doesn't execute the file method in Storage mock. This is not available in #google-cloud/storage but I did try to mock it. Is there a way to mock just the usage of google-cloud/storage using jest?
EDIT:
Here is the exception:
TypeError: str.bucket(...).file is not a function
at /home/vijaykumar/Documents/Code/Nestjs/cloud-storage-app/src/gcloud/bucket.operations.ts:37:6
at Generator.next (<anonymous>)
at /home/vijaykumar/Documents/Code/Nestjs/cloud-storage-app/src/gcloud/bucket.operations.ts:8:71
at new Promise (<anonymous>)
at Object.<anonymous>.__awaiter (/home/vijaykumar/Documents/Code/Nestjs/cloud-storage-app/src/gcloud/bucket.operations.ts:4:12)
at /home/vijaykumar/Documents/Code/Nestjs/cloud-storage-app/src/gcloud/bucket.operations.ts:22:40
at new Promise (<anonymous>)
at /home/vijaykumar/Documents/Code/Nestjs/cloud-storage-app/src/gcloud/bucket.operations.ts:22:9
Thanks to #ralemos. I was able to find the answer on how I mocked
Here is the complete implementation.
I've added a few more test stories as well
So jest.mock() esp the #google-cloud/storage modules, needs to be mocked in a different way. The Bucket of the Storage has all the details of the files in gcp storage, so that needs to be mocked first, I also mocked the File (this is of type #google-cloud/storage). Now I added the mockedFile to the mockedBucket and from there to the mockedStorage. I've also added all the methods and properties and implemented a mock for all of them.
There is a lodash node_module usage in my test file, so I mocked that implementation as well. Now everything works fine.
import { GcloudAuthenticationInstance } from '../common/services/gcloud.authentication';
const { Storage } = require('#google-cloud/storage');
const fs = require('fs');
import * as path from 'path';
import pump from 'pump';
import * as BucketOperations from './bucket.operations';
const { PassThrough } = require('stream');
const fsMock = jest.mock('fs');
const mockedFile = {
name: 'dev.txt',
createWriteStream: jest.fn().mockImplementation(() => {
return fs.createWriteStream(path.resolve(process.cwd(), './tests/cloud-storage/sample-write.txt'));
}),
createReadStream: jest.fn().mockImplementation(() => {
return fs.createReadStream(path.resolve(process.cwd(), './tests/cloud-storage/sample-read.txt'));
}),
};
jest.mock('lodash', () => {
return {
find: jest.fn().mockImplementation(() => {
return mockedFile;
})
};
});
const mockedBucket = {
file: jest.fn(() => mockedFile),
getFiles: jest.fn().mockImplementation(() => {
const fileArray = new Array();
fileArray.push(mockedFile);
return fileArray;
})
};
const mockedStorage = {
bucket: jest.fn(() => mockedBucket)
};
jest.mock('#google-cloud/storage', () => {
return {
Storage: jest.fn(() => mockedStorage)
};
});
jest.mock('pump', () => {
const mPump = { on: jest.fn() };
return jest.fn(() => mPump);
});
describe('Test suite for testing bucket operations', () => {
function cancelCloudStorageMock() {
jest.unmock('#google-cloud/storage');
jest.requireActual('#google-cloud/storage');
}
function cancelFsMock() {
jest.unmock('fs');
jest.requireActual('fs');
}
afterEach(() => {
jest.clearAllMocks();
//jest.restoreAllMocks();
});
test('test for uploadfiles - success', async (done) => {
pump().on = jest.fn(function(this: any, event, callback) {
if (event === 'finish') {
callback();
}
return this;
});
const actual = await BucketOperations.uploadEnvFiles('dev');
expect(actual).toEqual(
expect.objectContaining({
status: 'Success',
code: 201,
})
);
done();
});
test('test downloadEnvFiles - success', async (done) => {
jest.unmock('fs');
const downloadRes = await BucketOperations.downloadEnvFiles('dev');
expect(downloadRes).toBeDefined();
expect(downloadRes).toEqual(expect.objectContaining({code:200, status: 'Success'}));
done();
});
test('test for uploadfiles- failure', async (done) => {
cancelCloudStorageMock();
const bucketStorageSpy = jest
.spyOn(GcloudAuthenticationInstance, 'createGcloudAuthenticationBucket')
.mockImplementation(() => {
return new Storage({
projectId: 'testId',
keyFilename: path.resolve(process.cwd(), './tests/cloud-storage/sample-read.txt'),
scopes: ['testScope'],
autoRetry: false,
});
});
const mockReadable = new PassThrough();
const mockWritable = new PassThrough();
fs.createWriteStream = jest.fn().mockReturnValue(mockWritable);
fs.createReadStream = jest.fn().mockReturnValue(mockReadable);
pump().on = jest.fn(function(this: any, event, callback) {
if (event === 'error') {
callback();
}
return this;
});
const actual = BucketOperations.uploadEnvFiles('prod');
expect(actual).rejects.toEqual(
expect.objectContaining({
status: 'Error',
code: 500,
})
);
expect(bucketStorageSpy).toHaveBeenCalledTimes(1);
done();
});
test('test download - make the actual call - rej with auth error', async (done) => {
cancelCloudStorageMock();
console.dir(Storage);
const mockReadable = new PassThrough();
const mockWritable = new PassThrough();
fs.createWriteStream = jest.fn().mockReturnValue(mockWritable);
fs.createReadStream = jest.fn().mockReturnValue(mockReadable);
const createGcloudAuthenticationBucketSpy = jest
.spyOn(GcloudAuthenticationInstance, 'createGcloudAuthenticationBucket')
.mockImplementation(() => {
return new Storage();
});
try {
await BucketOperations.downloadEnvFiles('dev');
} catch (err) {
expect(err.code).toBe(500);
expect(err.status).toBe('Error');
}
expect(createGcloudAuthenticationBucketSpy).toHaveBeenCalledTimes(1);
createGcloudAuthenticationBucketSpy.mockReset();
done();
});
});

Migrating users from OneSignal to our own database that we push notifications from

I am trying to migrate our subscribers from OneSignal. I exported the endpoint, the keys (auth and P256DH) and I configured the VAPID keys of my OS account on my server.
When I try to send a notification from OS then remove the service worker of OS and use my own, it's sending the same notification that I previously sent through OS (quite odd), and when I programmatically remove the service worker of OS (through the console) and register my own service worker, it's responding with 410 error from chrome ("NotRegistered") and 401 from Firefox ("Request did not validate missing authorization header").
app.js file:
let isSubscribed = false;
let swRegistration = null;
let applicationKey = "PUBLIC_VAPID_KEY_HERE";
function urlB64ToUint8Array(base64String) {
const padding = '='.repeat((4 - base64String.length % 4) % 4);
const base64 = (base64String + padding)
.replace(/\-/g, '+')
.replace(/_/g, '/');
const rawData = window.atob(base64);
const outputArray = new Uint8Array(rawData.length);
for (let i = 0; i < rawData.length; ++i) {
outputArray[i] = rawData.charCodeAt(i);
}
return outputArray;
}
if ('serviceWorker' in navigator && 'PushManager' in window) {
console.log('Service Worker and Push is supported');
navigator.serviceWorker.register('sw.js')
.then(function (swReg) {
console.log('service worker registered');
swRegistration = swReg;
swRegistration.pushManager.getSubscription()
.then(function (subscription) {
isSubscribed = !(subscription === null);
if (isSubscribed) {
console.log('User is subscribed');
} else {
swRegistration.pushManager.subscribe({
userVisibleOnly: true,
applicationServerKey: urlB64ToUint8Array(applicationKey)
})
.then(function (subscription) {
console.log(subscription);
console.log('User is subscribed');
saveSubscription(subscription);
isSubscribed = true;
})
.catch(function (err) {
console.log('Failed to subscribe user: ', err);
})
}
})
})
.catch(function (error) {
console.error('Service Worker Error', error);
});
} else {
console.warn('Push messaging is not supported');
}
function saveSubscription(subscription) {
let xmlHttp = new XMLHttpRequest();
xmlHttp.open("POST", "/subscribe");
xmlHttp.setRequestHeader("Content-Type", "application/json;charset=UTF-8");
xmlHttp.onreadystatechange = function () {
if (xmlHttp.readyState != 4) return;
if (xmlHttp.status != 200 && xmlHttp.status != 304) {
console.log('HTTP error ' + xmlHttp.status, null);
} else {
console.log("User subscribed to server");
}
};
xmlHttp.send(JSON.stringify(subscription));
}
sw.js file:
let notificationUrl = '';
self.addEventListener('push', function (event) {
console.log('Push received: ', event);
let _data = event.data ? JSON.parse(event.data.text()) : {};
notificationUrl = _data.url;
event.waitUntil(
self.registration.showNotification(_data.title, {
body: _data.message,
icon: _data.icon,
tag: _data.tag
})
);
});
self.addEventListener('notificationclick', function (event) {
event.notification.close();
event.waitUntil(
clients.matchAll({
type: "window"
})
.then(function (clientList) {
if (clients.openWindow) {
return clients.openWindow(notificationUrl);
}
})
);
});
push.js file which pushes notifications:
const express = require('express');
const router = express.Router();
const q = require('q');
const webPush = require('web-push');
const keys = require('./../config/keys');
const mysql = require("mysql");
const pool = mysql.createPool({
connectionLimit: 10,
host: 'localhost',
user: 'root',
password: 'root',
database: 'webpush',
multipleStatements: true,
dateStrings: true
});
router.post('/push', (req, res) => {
const payload = {
title: req.body.title,
message: req.body.message,
url: req.body.url,
ttl: req.body.ttl,
icon: req.body.icon,
image: req.body.image,
badge: req.body.badge,
tag: req.body.tag
};
pool.query('SELECT * FROM subscriber', (err, subscriptions) => {
if (err) {
return console.log(err);
console.error(`Error occurred while getting subscriptions`);
return res.status(500).json({
error: 'Technical error occurred'
});
}
if (!subscriptions.length) {
console.error(`No subscribers found`);
return res.status(500).json({
error: 'Subscribers not found'
});
}
let parallelSubscriptionCalls = subscriptions.map(subscription => {
return new Promise((resolve, reject) => {
const pushSubscription = {
endpoint: subscription.endpoint,
keys: {
p256dh: subscription.p256dh,
auth: subscription.auth
}
};
const pushPayload = JSON.stringify(payload);
const pushOptions = {
vapidDetails: {
subject: 'https://www.mydomainhere.com',
privateKey: keys.privateKey,
publicKey: keys.publicKey
},
TTL: payload.ttl,
headers: {}
};
webPush.sendNotification(pushSubscription, pushPayload, pushOptions)
.then((value) => {
resolve({
status: true,
endpoint: subscription.endpoint,
data: value
});
}).catch((err) => {
reject({
status: false,
endpoint: subscription.endpoint,
data: err
});
});
});
});
q.allSettled(parallelSubscriptionCalls).then((pushResults) => {
console.info(pushResults);
});
res.json({
data: 'Push triggered'
});
})
});
module.exports = router;
subscribe.js file which does the subscription:
const express = require('express');
const router = express.Router();
const mysql = require("mysql");
const pool = mysql.createPool({
connectionLimit: 10,
host: 'localhost',
user: 'root',
password: 'root',
database: 'webpush',
multipleStatements: true,
dateStrings: true
});
router.post('/subscribe', (req, res) => {
const endpoint = req.body.endpoint;
const auth = req.body.keys.auth;
const p256dh = req.body.keys.p256dh;
const subscriptionSet = { endpoint, auth, p256dh }
pool.getConnection((err, connection) => {
if (err) {
console.error(`Error occurred while saving subscription. Err: ${err}`);
return res.status(500).json({
error: 'Technical error occurred'
});
};
connection.query('INSERT INTO subscriber SET ?', subscriptionSet, (err, subscription) => {
if (err) {
console.error(`Error occurred while saving subscription. Err: ${err}`);
return res.status(500).json({
error: 'Technical error occurred'
});
}
res.json({
data: 'Subscription saved.'
})
})
});
});
module.exports = router;
Im trying to do the same, how and where did you export the subscribers auth and P256DH from onesignal? because in the csv export, onesignal provides the push_token (endpoint) and not the auth and P256DH. Where would i get the all the users auth and P256DH?
From my understanding you cannot re register the service worker unless the subscriber comes back and visits your domain. However, the service worker updates every day, so what you can do is edit the existing One signal service worker files - delete their content and add your own code or import scripts. Make sure not to change file location or rename the one signal service worker file, it needs to be the same filename. This will count as an 'update' and browsers should automatically replace the contents without any action from users. If there's two OneSignal service worker files then change both.

Resources