Sequelize: Not all values in forEach get inserted to SQLite DB - node.js

Question

I have an sqlite DB which i want to populate with 137k entries which are retrieved from multiple API calls.
There are only about 6k entries in my sqlite db, but if I export the lines into a text file, in the text file is the correct amount of 137k lines.
Here is the code for inserting into the database and appending to the text file.
Here is my index.js
const db = require("./config/database");
const items = require("./models/items");
var fs = require('fs');
const baseURL = 'xxx';
const namespace = 'static-eu';
const region = 'eu';
const client = 'xxx';
const secret = 'xxx';
const tokenURL = 'xxx';
const getToken = async function(client, secret)
{
const body = "grant_type=client_credentials";
const response = await fetch(tokenURL, {
method: 'post',
body: body,
headers: {'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'Basic ' + Buffer.from(client + ':' + secret).toString('base64')}
});
const data = await response.json();
const jsondata = JSON.stringify(data);
const json = JSON.parse(jsondata);
return json['access_token'];
}
const getItems = async function(ID)
{
const access_token = await getToken(client, secret);
const reqURL = baseURL + '/data/wow/search/item?namespace=' + namespace + '&_pageSize=1000&orderby=id&id=['+ ID + ',]&access_token=' + access_token;
const response = await fetch(reqURL, {
method: 'get',
//headers: {'Authorization': 'Bearer ' + access_token}
});
const data = await response.json();
const jsondata = JSON.stringify(data);
const json = JSON.parse(jsondata);
var arrayID = [];
json.results.forEach(function (item, index){
try {
items.upsert({
itemID: json.results[index].data.id,
desc_de: json.results[index].data.name.de_DE,
desc_en: json.results[index].data.name.en_US,
media: json.results[index].data.id,
item_class_de: json.results[index].data.item_class.name.de_DE,
item_class_en: json.results[index].data.item_class.name.en_US,
});
}
catch (error) {
return console.log(error.name + error.message);
}
arrayID.push(json.results[index].data.id);
});
const nextID = (Math.max(...arrayID)) + 1;
return nextID;
//Object.keys(json.results[data]).forEach(function(results){
// console.log('Key : ' + results + ', Value : ' + json["id"])
// })
}
const getAll = async function()
{
let ID = 1;
while(1==1)
{
console.log(ID);
ID = await getItems(ID);
}};
db.authenticate()
.then(() => {
console.log('Logged in to DB!');
items.init(db);
items.sync();
})
.catch(err => console.log(err));
getAll();
my items.js
const { DataTypes, Model } = require('sequelize');
module.exports = class items extends Model {
static init(sequelize){
return super.init({
itemID: {
type: DataTypes.INTEGER,
allowNull: false,
unique: true
},
desc_de: DataTypes.STRING,
desc_en: DataTypes.STRING,
media: DataTypes.INTEGER,
item_class_de: DataTypes.STRING,
item_class_en: DataTypes.STRING,
}, {
tableName: 'items',
timestamps: true,
sequelize
});
}
}
my database.js
const { Sequelize } = require('sequelize');
module.exports = new Sequelize('database', 'user', 'password', {
host: 'localhost',
dialect: 'sqlite',
logging: false,
storage: './data/items.sqlite',
});

Answer 1

More context would be required for this one.
What you could do is check if items.upsert() returns any errors.
Add a try-catch block to check for any issues.
Use a loop instead to insert the data into batches. It definitely improves performance, a good library is async or bluebird
Check your db schema if it can handle 137k entries.
Edit: Read these updated codes out and see how it fits in your requirements
getAll()
const pQueue = require('p-queue');
const queue = new pQueue({ concurrency: 5 });
const getAll = async function() {
let ID = 1;
while (true) {
console.log(ID);
const nextID = await queue.add(() => getItems(ID));
if (!nextID) {
break;
}
ID = nextID;
}
};
getItems()
const getItems = async function(ID) {
const access_token = await getToken(client, secret);
const reqURL =
baseURL +
'/data/wow/search/item?namespace=' +
namespace +
'&_pageSize=1000&orderby=id&id=[' +
ID +
',]&access_token=' +
access_token;
const response = await fetch(reqURL, {
method: 'get',
});
const data = await response.json();
const jsondata = JSON.stringify(data);
const json = JSON.parse(jsondata);
var arrayID = [];
json.results.forEach(function(item, index) {
try {
items.upsert({
itemID: json.results[index].data.id,
desc_de: json.results[index].data.name.de_DE,
desc_en: json.results[index].data.name.en_US,
media: json.results[index].data.id,
item_class_de: json.results[index].data.item_class.name.de_DE,
item_class_en: json.results[index].data.item_class.name.en_US,
});
} catch (error) {
return console.log(error.name + error.message);
}
arrayID.push(json.results[index].data.id);
});
if (!json.hasOwnProperty('next_page')) {
return false;
}
const nextID = Math.max(...arrayID) + 1;
return nextID;
};
Alternate version with transactions
const getItems = async function(ID) {
const access_token = await getToken(client, secret);
const reqURL =
baseURL +
'/data/wow/search/item?namespace=' +
namespace +
'&_pageSize=1000&orderby=id&id=[' +
ID +
',]&access_token=' +
access_token;
const response = await fetch(reqURL, {
method: 'get',
});
const data = await response.json();
const jsondata = JSON.stringify(data);
const json = JSON.parse(jsondata);
var arrayID = [];
const itemsData = json.results.map(function(item) {
return {
itemID: item.data.id,
desc_de: item.data.name.de_DE,
desc_en: item.data.name.en_US,
media: item.data.id,
item_class_de: item.data.item_class.name.de_DE,
item_class_en: item.data.item_class.name.en_US,
};
});
return db.transaction(async (transaction) => {
await items.bulkCreate(itemsData, { transaction });
}).then(() => {
arrayID = json.results.map(function(item) {
return item.data.id;
});
if (!json.hasOwnProperty('next_page')) {
return false;
}
const nextID = Math.max(...arrayID) + 1;
return nextID;
});
};