Sails v1.x stores empty properties in database mongoDb, when POST api call is made with request having few properties only from actual model defined in sails model, such properties which are not available in request is saved with empty values in database collection which was not the case in sails v0.12
Since I have migrated project from sails v0.12 to sails v1.x, I am facing this issue which is leading to code breakage on api response.
I am using JOI library to validate the request properties and below is the data that is being sent to <ModelName>.create(topic).fetch()
topic = {
published: false,
closed: false,
name: 'Topic G -01',
communityId: '57d640b3-cda6-4703-8080-2ef604de2086',
activityId: '23ef04f4-699d-47c7-94ad-3fb5b1c18889',
createdByUserId: '3cc6dddd-9425-42b8-83fc-4ba3f068128d',
board: '5f7cc128c2380d00172225ee'
}
Data getting saved in database collection as below:
{
_id: ObjectId("61c9d4ec5ceca05c4b0016cb"),
published: false,
closed: false,
name: 'Topic G -01',
communityId: '57d640b3-cda6-4703-8080-2ef604de2086',
activityId: '23ef04f4-699d-47c7-94ad-3fb5b1c18889',
createdByUserId: '3cc6dddd-9425-42b8-83fc-4ba3f068128d',
board: '5f7cc128c2380d00172225ee',
createdAt: '2021-12-28T16:11:07.229Z',
updatedAt: '2021-12-28T16:11:07.229Z',
description: '',
content: null,
official: false,
locked: false,
singleResponses: false,
responseTierLimit: 0,
deleteOwnResponses: false,
showQuantResults: false,
probeQuantResponse: false,
ratingSystem: '',
responseVisibility: '',
allowParticipantQuestions: false,
qIndexVisibility: false,
disablePostReplies: false,
publishedAt: null,
questionIds: null,
updatedByUserId: '',
includedRoles: null,
}
However in sails v0.12, on the contrary it used to store as below:
{
_id: ObjectId("61c9d4ec5ceca05c4b0016cb"),
published: false,
closed: false,
name: 'Topic G -01',
communityId: '57d640b3-cda6-4703-8080-2ef604de2086',
activityId: '23ef04f4-699d-47c7-94ad-3fb5b1c18889',
createdByUserId: '3cc6dddd-9425-42b8-83fc-4ba3f068128d',
board: '5f7cc128c2380d00172225ee',
createdAt: '2021-12-28T16:11:07.229Z',
updatedAt: '2021-12-28T16:11:07.229Z',
}
Related
I am trying to save data from tweets to a mongoDB database using node and express.
I am using the twitter api to stream twitter data with a specific hashtags. I just want to save the text content of the post:
Here is how the tweet content shows up when it is console.logged:
(Note this feature works and this is my own posted
{
created_at: 'Tue Mar 15 06:38:58 +0000 2022',
id: 1503621761388134400,
id_str: '1503621761388134410',
text: '#TelecomDisaster Test for project 2',
source: 'Twitter Web App',
truncated: false,
in_reply_to_status_id: null,
in_reply_to_status_id_str: null,
in_reply_to_user_id: null,
in_reply_to_user_id_str: null,
in_reply_to_screen_name: null,
user: {
id: 1472188612494172200,
id_str: '1472188612494172172',
name: 'Dillon Rampersad',
screen_name: 'R_Dillon_25',
location: null,
url: null,
description: null,
translator_type: 'none',
protected: false,
verified: false,
followers_count: 5,
friends_count: 11,
listed_count: 0,
favourites_count: 22,
statuses_count: 63,
created_at: 'Sat Dec 18 12:55:26 +0000 2021',
utc_offset: null,
time_zone: null,
geo_enabled: false,
lang: null,
contributors_enabled: false,
is_translator: false,
profile_background_color: 'F5F8FA',
profile_background_image_url: '',
profile_background_image_url_https: '',
profile_background_tile: false,
profile_link_color: '1DA1F2',
profile_sidebar_border_color: 'C0DEED',
profile_sidebar_fill_color: 'DDEEF6',
profile_text_color: '333333',
profile_use_background_image: true,
profile_image_url: 'http://pbs.twimg.com/profile_images/1472188757956780033/OMlZZeZI_normal.jpg',
profile_image_url_https: 'https://pbs.twimg.com/profile_images/1472188757956780033/OMlZZeZI_normal.jpg',
default_profile: true,
default_profile_image: false,
following: null,
follow_request_sent: null,
notifications: null,
withheld_in_countries: []
},
geo: null,
coordinates: null,
place: null,
contributors: null,
is_quote_status: false,
quote_count: 0,
reply_count: 0,
retweet_count: 0,
favorite_count: 0,
entities: { hashtags: [ [Object] ], urls: [], user_mentions: [], symbols: [] },
favorited: false,
retweeted: false,
filter_level: 'low',
lang: 'en',
timestamp_ms: '1647326338513'
}
I want to save text: '#TelecomDisaster Test for project 2', and created_at: 'Tue Mar 15 06:38:58 +0000 2022', to my database.
I am trying with the function below to save just the text for now but i dont quite understand how to:
const express = require('express')
const router = new express.Router();
var Twitter = require('twit')
const TwitterPosts = require("../db/models/TwitterPosts.model");
//api keys goes here but it removed for safety
var stream = client.stream('statuses/filter', { track: '#TelecomDisaster' })
stream.on('tweet', function (tweet) {
console.log(tweet)
let newtweet = new TwitterPosts({
tweet: req.body.postContent
});
newtweet.save().then((twit) => {
res.send(twit);
console.log(twit);
})
});
module.exports = router;
The model for the schema:
const mongoose = require('mongoose');
const TwitterPostsSchema = new mongoose.Schema({
twitterUsername:{
type: String,
required: false,
minlength:1,
trim: true
},
postContent:{
type: String,
required: false,
minlength:1,
trim: true
},
postDateTime:{
type: Date,
required: false,
default: Date.now
}
})
const TwitterPosts = mongoose.model( 'TwitterPosts', TwitterPostsSchema);
module.exports = TwitterPosts
Whenever it trys to save i get the error
tweet: req.body.postContent
^
ReferenceError: req is not defined
i did not define req but in this use case i dont know how to do that exactly when streaming the tweets.
To conclude i am trying to save tweets to a mongoDB database using node and express. the tweets are streamed as shown above but i dont quite understand how it is saved to the database.
you receive tweet in stream.on listener, so it's just tweet, instead of req.body.postContent:
let newtweet = new TwitterPosts({
tweet: tweet
});
or, according to your schema:
let newtweet = new TwitterPosts({
twitterUsername: tweet.user.name,
postContent: tweet.text,
postDateTime: tweet.created_at
});
This question already has answers here:
How do you turn a Mongoose document into a plain object?
(9 answers)
Closed 2 years ago.
There I have res is original object
{
time: 2020-07-26T10:39:38.135Z,
isTransfered: true,
_id: 5f1d5d6b60755e75b48770a6,
receiverAccountNumber: '12345678',
transfererAccountNumber: '11111111',
receiverName: 'Lê Công Tuyền',
transfererName: 'Nguyễn Thanh Tuấn',
amount: 1000000,
content: "test chuyefo'seajfg",
payFeeBy: 'transferer',
type: { name: 'internal', bankCode: 'TUB' },
__v: 0
}
And I got this result (called res2) is returned object using spread operator res2 = {...res} :
{
'$__': InternalCache {
strictMode: true,
selected: {},
// alot of key-value
'$setCalled': Set(0) {},
ownerDocument: undefined,
fullPath: undefined,
emitter: EventEmitter {
_events: [Object: null prototype],
_eventsCount: 2,
_maxListeners: 0,
[Symbol(kCapture)]: false
},
'$options': { skipId: true, isNew: false, willInit: true }
},
isNew: false,
errors: undefined,
_doc: {
time: 2020-07-26T10:39:38.135Z,
isTransfered: true,
_id: 5f1d5d6b60755e75b48770a6,
receiverAccountNumber: '12345678',
transfererAccountNumber: '11111111',
receiverName: 'Lê Công Tuyền',
transfererName: 'Nguyễn Thanh Tuấn',
amount: 1000000,
content: "test chuyefo'seajfg",
payFeeBy: 'transferer',
type: { name: 'internal', bankCode: 'TUB' },
__v: 0
},
'$locals': {},
'$op': null,
'$init': true
}
I really dont know about this behavior of spread operator, that a lot of new key-value generated and the object I want to get is in _doc key.
Code is run on Nodejs v12 (nvm use 12)
This is probably because the res here is a mongoose document which has these values.
When you are doing a mongoose query by default it returns a document object. In order to receive a plain object use lean(). If you use lean then you won't be getting these unnecessary data when using spread operator.
Schema.findOne(query).lean()
However, if you need a document object from mongoose then in this case you can try the following to get rid of other values you don't need.
let res2 = { ...res.toObject() };
I have a Sequelize model called Test with a unique field value (primary key). I'm using SQLite as the database management system.
If I use the bulkCreate() option ignoreDuplicates: true, then bulkCreate will ignore the new data if they already exist in the database. That works great, but the objects returned by bulkCreate() always have isNewRecord: false, even when a new record was inserted into the database.
Code
const items = [ {value: 'a'}, {value: 'b'} ]; // Items to be saved in database.
const results = await Test.bulkCreate(items, {
ignoreDuplicates: true // Ignore duplicate records
});
After the first execution of bulkCreate() and the database empty, the value of results:
[
Test {
dataValues: {
value: 'a',
createdAt: 2020-07-12T12:01:08.695Z,
updatedAt: 2020-07-12T12:01:08.695Z
},
_previousDataValues: {
value: 'a',
createdAt: 2020-07-12T12:01:08.695Z,
updatedAt: 2020-07-12T12:01:08.695Z
},
_changed: Set {},
_options: {
isNewRecord: true,
_schema: null,
_schemaDelimiter: '',
include: undefined
},
isNewRecord: false
},
Test {
dataValues: {
value: 'b',
createdAt: 2020-07-12T12:01:08.695Z,
updatedAt: 2020-07-12T12:01:08.695Z
},
_previousDataValues: {
value: 'b',
createdAt: 2020-07-12T12:01:08.695Z,
updatedAt: 2020-07-12T12:01:08.695Z
},
_changed: Set {},
_options: {
isNewRecord: true,
_schema: null,
_schemaDelimiter: '',
include: undefined
},
isNewRecord: false
}
]
I expected isNewRecord: true after the first execution. What am I missing?
Environment
Windows 10 Pro
NodeJS v12.16.2
sequelize: 6.3.3
sqlite3: 5.0.0
Sequelize docs
isNewRecord of Model:
ignoreDuplicates of bulkCreate():
You need to access _options.isNewRecord.
As this issue states that the isNewRecord only be set to true if the instance has not yet been persisted to the database.
In your case, because you have saved the instance (the first time or second time, it doesn't matter, you already save it) so isNewRecord is set to false.
So as I mention above, if you want to check if your instance is just saved the first time or not, access _options.isNewRecord.
The isNewRecord field is true if the Model instance does not exist in the database.
const user = new User({ ...userData });
console.log(user.isNewRecord); // true
await user.save();
console.log(user.isNewRecord); // false
I have a web application that is listening to webhook to the accounts they are connected to via oauth and for some reason every webhook that is coming in doesn't have the user_id attribute on them.
{
id: 'evt_18USWeLQuocIBkVl4x3RYTYn',
object: 'event',
api_version: '2016-06-15',
created: 1467859564,
data:
{ object:
{ id: 'ch_18USWdLQuocIBkVlfev2xN9n',
object: 'charge',
amount: 300,
amount_refunded: 0,
application_fee: null,
balance_transaction: 'txn_18USWeLQuocIBkVlYKlhfYl7',
captured: true,
created: 1467859563,
currency: 'usd',
customer: null,
description: '',
destination: null,
dispute: null,
failure_code: null,
failure_message: null,
fraud_details: {},
invoice: null,
livemode: false,
metadata: {},
order: null,
paid: true,
receipt_email: null,
receipt_number: null,
refunded: false,
refunds: [Object],
shipping: null,
source: [Object],
source_transfer: null,
statement_descriptor: null,
status: 'succeeded' } },
livemode: false,
pending_webhooks: 2,
request: 'req_8m0X3GgVB7IWpM',
type: 'charge.succeeded'
}
There are two different types of webhooks that can be set up:
"Account" webhooks will receive events that happen on your own account
"Connect" webhooks will receive events that happen on any account that is connected to your platform
Only events sent to "Connect" webhooks will include the user_id field.
If you're receiving events without a user_id field, then that means the event happened on your own account, and was sent to you via an "Account" endpoint rather than a "Connect" endpoint.
I'm trying to produce a lean record for a REST API that include virtual fields.
The official documentation for how to implement virtual fields for Mongoose:
http://mongoosejs.com/docs/guide.html
My model:
var keystone = require('keystone')
, Types = keystone.Field.Types
, list = new keystone.List('Vendors');
list.add({
name : {
first: {type : Types.Text}
, last: {type : Types.Text}
}
});
list.schema.virtual('name.full').get(function() {
return this.name.first + ' ' + this.name.last;
});
list.register();
Now, let's query the model:
var keystone = require('keystone'),
vendors = keystone.list('Vendors');
vendors.model.find()
.exec(function(err, doc){
console.log(doc)
});
Virtual field name.full is not here:
[ { _id: 563acf280f2b2dfd4f59bcf3,
__v: 0,
name: { first: 'Walter', last: 'White' } }]
But if we do this:
vendors.model.find()
.exec(function(err, doc){
console.log(doc.name.full); // "Walter White"
});
Then the virtual shows.
I guess the reason is that when I do a console.log(doc) the Mongoose document.toString() method is invoked which does not include virtuals by default. Fair enough. That's understandable.
To include the virtuals in any of the conversion methods you have to go:
doc.toString({virtuals: true})
doc.toObject({virtuals: true})
doc.toJSON({virtuals: true})
However, this includes keys I don't want for my REST API to pump out to my users:
{ _id: 563acf280f2b2dfd4f59bcf3,
__v: 0,
name: { first: 'Walter', last: 'White', full: 'Walter White' },
_: { name: { last: [Object], first: [Object] } },
list:
List {
options:
{ schema: [Object],
noedit: false,
nocreate: false,
nodelete: false,
autocreate: false,
sortable: false,
hidden: false,
track: false,
inherits: false,
searchFields: '__name__',
defaultSort: '__default__',
defaultColumns: '__name__',
label: 'Vendors' },
key: 'Vendors',
path: 'vendors',
schema:
Schema {
paths: [Object],
subpaths: {},
virtuals: [Object],
nested: [Object],
inherits: {},
callQueue: [],
_indexes: [],
methods: [Object],
statics: {},
tree: [Object],
_requiredpaths: [],
discriminatorMapping: undefined,
_indexedpaths: undefined,
options: [Object] },
schemaFields: [ [Object] ],
uiElements: [ [Object], [Object] ],
underscoreMethods: { name: [Object] },
fields: { 'name.first': [Object], 'name.last': [Object] },
fieldTypes: { text: true },
relationships: {},
mappings:
{ name: null,
createdBy: null,
createdOn: null,
modifiedBy: null,
modifiedOn: null },
model:
{ [Function: model]
base: [Object],
modelName: 'Vendors',
model: [Function: model],
db: [Object],
discriminators: undefined,
schema: [Object],
options: undefined,
collection: [Object] } },
id: '563acf280f2b2dfd4f59bcf3' }
I can always of course just delete the unwanted keys, but this doesn't seem quite right:
vendors.model.findOne()
.exec(function(err, doc){
var c = doc.toObject({virtuals: true});
delete c.list;
delete c._;
console.log(c)
});
This produces what I need:
{ _id: 563acf280f2b2dfd4f59bcf3,
__v: 0,
name: { first: 'Walter', last: 'White', full: 'Walter White' },
id: '563acf280f2b2dfd4f59bcf3' }
Is there not a better way of getting a lean record?
I think you want the select method.. something like this:
vendors.model.findOne()
.select('_id __v name').
.exec(function(err, doc){
console.log(c)
});
Also personally I prefer setting virtuals: true on the schema rather than the document, but depends on use case I guess.
One solution would be to use a module like Lodash (or Underscore) which allows you pick a whitelist of property names:
vendors.model.findOne()
.exec(function(err, doc){
var c = _.pick(doc, ['id', 'name.first', 'name.last', 'name.full']);
console.log(c)
});
Given your use-case of serving this data via REST API, I think explicitly defining a whitelist of property names is safer. You could even define a virtual property on your schema which returns the predefined whitelist:
list.schema.virtual('whitelist').get(function() {
return ['id', 'name.first', 'name.last', 'name.full'];
});
and use it in multiple places, or have different versions of your whitelist, all managed at the model layer.