I have a mongodb collection which has a large quantity of entries. + 10000.
All of these have the same paramters:
Val_string
T1_string
T2_string
I am receiving this via a get request from my mobile app however the order is not correct.
How can I sort my collection by the Val string which is a number going from 1 to 10000 in this case.
I am using mongoose mongodb and node js.
Presently I query the collection and get 5 results send them and then loop another 5 until I have read all the results.
So I need to sort the collection out prior to sending this data.
app.get("/testmethod", function(req, res)
{
mongoose.connect("mongodb://localhost:27017/CTI", {useNewUrlParser: true},function(err, client){
if(err) {
console.log(err)
}
else
{
client.db.collection("datas").find().limit(items_to_send).skip(Sent_data).toArray(function(err, result) {
if (err) {throw err;}
//console.log(result);
Sent_data = Sent_data + items_to_send;
db.close();
var D1_T1 = result[0].T1_String;
var D1_T2 = result[0].T2_String;
var D2_T1 = result[1].T1_String;
var D2_T2 = result[1].T2_String;
var D3_T1 = result[2].T1_String;
var D3_T2 = result[2].T2_String;
var D4_T1 = result[3].T1_String;
var D4_T2 = result[3].T2_String;
var D5_T1 = result[4].T1_String;
var D5_T2 = result[4].T2_String;
res.status(200).send({
D1_T1: D1_T1,
D1_T2: D1_T2,
D2_T1: D2_T1,
D2_T2: D2_T2,
D3_T1: D3_T1,
D3_T2: D3_T2,
D4_T1: D4_T1,
D4_T2: D4_T2,
D5_T1: D5_T1,
D5_T2: D5_T2
});
});
}//End of else
});//End of connect
});
What I want is a sort to order the entire collection by the val string field which goes from 1-10000
You have as below
client.db.collection("datas").find().limit(items_to_send).skip(Sent_data)
You can add .sort({fieldName:sortOrder})
sort order: 1 / -1(desc)
client.db.collection("datas").find().limit(items_to_send)
.skip(Sent_data).sort({"val_string":1})
Sort follows lexicographical order so no problem of being string.
Related
currently i have data that needs to be inserted to mongodb. The data have been successfully inserted into mongodb, however there is some value that i would like to add to the data and append it into mongodb.
How can i do so? This is my code for inserting data into mongodb
var MongoClient = require('mongodb').MongoClient
, format = require('util').format;
MongoClient.connect('mongodb://127.0.0.1:27017/test', function(err, db) {
if(err) throw err;
var collection = db.collection('test_insert');
collection.insert({ values:parsestring() }, function(err, docs) {
collection.count(function(err, count) {
console.log(format("count = %s", count));
});
});
// Locate all the entries using find
collection.find().toArray(function(err, results) {
console.dir(results);
// Let's close the db
db.close();
});
});
// "1,61,54,87,20,12/3/2016,8:39AM" this default value
function parsestring(str="1,30,40,50,20,10/10/2016,10:39AM")
{
return str.split(",");
}
I would like to add value to the text string.
For example:
Machine Unit: 1,
Air Temperature °C: 30,
Water Temperature °C: 40,
Heat Temperature °C: 50,
Room Temperature °C: 20,
Date: 10/10/2016,
Time: 10:39AM
Like already mentioned in the comments, you should store your data as objects (or arrays). The line str.split(",") already returns you an array, which you store. In your code you also fetch your data as an Array. When you console.dir() your results, you could map your results.entries to a specific output string if you prefer.
collection.find().toArray(function(err, results) {
console.dir(results); // <= results.entries
// Let's close the db
db.close();
});
See also:
https://developer.mozilla.org/de/docs/Web/JavaScript/Reference/Global_Objects/Array/map &
https://developer.mozilla.org/de/docs/Web/JavaScript/Reference/Global_Objects/Array/entries
However, if you want to store objects (e.g. for accessing data via identifier) you could simply do it like:
function parsestring(str="1,30,40,50,20,10/10/2016,10:39AM"){
var dataArr = str.split(",");
var dbEntry = {};
dbEntry.machine = dataArr[0];
dbEntry.airTemp = dataArr[1];
dbEntry.waterTemp = dataArr[2];
dbEntry.heatTemp = dataArr[3];
dbEntry.roomTemp = dataArr[4];
dbEntry.date = dataArr[5];
dbEntry.time = dataArr[6];
return dbEntry;
}
When returning a simple object from mongoDB, you won't need .toArray()
I'm trying to connect to a mongoDB DB, and make some processes on a collection, and close the collection when all the collection items were processed. when I'm trying to receive array.length, I get undefined.
Db = require('mongodb').Db;
Server = require('mongodb').Server;
const db = new Db(DB_NAME, new Server(HOST, PORT));
// connect to mongoDB
db.open(function (err, db) {
const Collection = db.collection(COLLECTION_NAME);
var items = Collection.find({});
var itemsLength = items.lebgth;
var itemsProcessed = 0;
items.forEach((item, index, array) => {
// some process like:
Collection.update({query}, {set}, callback)
itemsProcessed++;
if(itemsProcessed == array.length){
db.close();
// close connection if all items were processed
}
});
});
Is there any other way to do it?
Just use db.collection.count()
EDIT:
You can use the optional callback of forEach, as documented here:
items.forEach((item) => {
// some process...
}, (err) => db.close()); // close connection if all items were processed
check this!
Collection.stat()
In the documentation of Algolia, for the node.js part they specified to use MySQL for indexing but not MongoDB, I have another question regarding this issue but it is more a general question , check here
Some folks ask me to use mongo-connector but tried it and I got some unknown error, which got me to square one
My real question is, how do i iterate a list of collections in mongodb to algolia?
This is the Algolia's version of MySQL in Node.js
var _ = require('lodash');
var async = require('async');
var mysql = require('mysql');
var algoliasearch = require('algoliasearch');
var client = algoliasearch("RQGLD4LOQI", "••••••••••••••••••••••••••••••••");
var index = client.initIndex('YourIndexName');
var connection = mysql.createConnection({
host: 'localhost',
user: 'mysql_user',
password: 'mysql_password',
database: 'YourDatabaseName'
});
connection.query('SELECT * FROM TABLE_TO_IMPORT', function(err, results, fields) {
if (err) {
throw err;
}
// let's use table IDS as Algolia objectIDs
results = results.map(function(result) {
result.objectID = result.id;
return result;
});
// split our results into chunks of 5,000 objects, to get a good indexing/insert performance
var chunkedResults = _.chunk(results, 5000);
// for each chunk of 5,000 objects, save to algolia, in parallel. Call end() when finished
// or if any save produces an error
// https://github.com/caolan/async#eacharr-iterator-callback
async.each(chunkedResults, index.saveObjects.bind(index), end);
});
function end(err) {
if (err) {
throw err;
}
console.log('MySQL<>Algolia import done')
};
To be specific I'm using mongoose as my ORM, so I have no experience in other libraries. Please help me on this, so that I could some searching interface already :(.
You can use the following code to iterate over the whole MongoDB mydb.myCollection collection + create batches that will be sent to the Algolia index:
var Db = require('mongodb').Db,
Server = require('mongodb').Server,
algoliasearch = require('algoliasearch');
// init Algolia index
var client = algoliasearch("*********", "••••••••••••••••••••••••••••••••");
var index = client.initIndex('YourIndexName');
// init connection to MongoDB
var db = new Db('mydb', new Server('localhost', 27017));
db.open(function(err, db) {
// get the collection
db.collection('myCollection', function(err, collection) {
// iterate over the whole collection using a cursor
var batch = [];
collection.find().forEach(function(doc) {
batch.push(doc);
if (batch.length > 10000) {
// send documents by batch of 10000 to Algolia
index.addObjects(batch);
batch = [];
}
});
// last batch
if (batch.length > 0) {
index.addObjects(batch);
}
});
});
I've tried to find a solution looking at similar problems. One main issue that I've come across is that the Date is in a String format or not in the some altered form. I have a large file of data in JSON format. First I read through that file and convert each String to a date OBJ , the push all my data to my MongoDB.
convert to date.
var fs = require('fs');
fs.readFile('data.txt',function(err,data){
var x = [];
if(err) throw err;
var buf = new Buffer(data);
x = buf.toString();
x = JSON.parse(x);
//console.log(x);
for(var a in x){
x[a].TrxDate=new Date(x[a].TrxDate);
}
x = JSON.stringify(x);
fs.writeFile('data.txt',x,callback());
});
function callback(){
console.log('finished');
}
my date field in the DB
"TxDate": "2015-01-01T05:00:00.000Z"
Here is my query. It returns nothing found
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var url = 'blank';
MongoClient.connect(url,function(err,db){
if(err){
console.log('unable to connect to the mongodb server Error:',err);
}else {
console.log('connection good:',url);
var collection = db.collection('records');
collection.find({TxDate: { $gte: new Date('2015-07-01')}}).toArray(function(err,result){
if(err){
console.log(err);
}else if (result.length){
console.log('Found',result);
}else{
console.log('Nothing found');
}
});
}
});
What am I doing wrong here? Something I just thought of, Is it possible for me to check the type of the object int the DB?
I am new to nodejs and I am trying to fetch some data from my PG server. I manage to get my data but not in the order I expected. I may not use it the proper way, can anyone help ?
Here is a sample of code :
var pg = require('pg');
var db = new pg.Client(conString);
var link = db.connect();
var data = {};
// -----
console.log(prefix+'Fetching categories');
db.query('SELECT DISTINCT category FROM cc WHERE category IS NOT NULL', function(err, data){
data.rows.forEach(function(row){
data[row.category] = {}; // initialise
});
console.log('1111111',data,'---------');
});
console.log('2222222',data,'---------');
for (var category in data)
{
console.log(prefix+'Listing values for on "'+category+'"');
var values = db.query('SELECT SUBSTRING(date::varchar, 1,7) AS month, sum(amount) FROM cc WHERE category = \''+category+'\' GROUP BY 1 ORDER BY 1', function(err, data){
console.log('Got values',data.rows);
});
}
// -----
console.log(prefix+'Ending connection to database');
// db.end();
// -----
console.log(prefix+'Ending transaction on server side');
response.end();
I get 222222 before 1111111 :/ so my result is sent as empty and then it is filled :( How do I have to do ?
Thanks for your time !
Take a look at promises :https://github.com/promises-aplus/promises-spec. (one good lib for nodejs is q - http://documentup.com/kriskowal/q/).they are really useful in keeping nodejs code coherent and neat in all those callbacks.a must for every nodejs developer.