How to append on existing field with elasticsearch python? - python-3.x

I am using kibana and Elasticsearch version 5.1.1 and python version 3.6.
I have created my index like this
put_books
The function to add a user is this one :
def add_user(first_name, last_name, age, mail):
doc = {"first_name": "" + first_name, "last_name": "" + last_name, "age": age, "email": "" + mail}
global id_user
res = es.index(index="books", doc_type="user", id=id_user, body=doc)
id_user += 1
print(res['result'])
and to add preferences :
def add_preferences(preferences, i):
doc = es.get(index="books", doc_type="user", id=id_book)
res = es.update(index="books", doc_type="user", id=i, body={'doc':{"keyword_preferences": preferences}})
My problem is here : when I want to add preferences, it success but if I want to add again preferences, it replace it :
id_user = 1
nom = "nom_1"
prenom = "prenom_1"
age = 45
email = "adresse_mail_1"
add_user(prenom, nom, age, email)
add_preferences("comique", 1)
add_preferences("horreur", 1)
get_user(1)
the result is :
updated
{'first_name': 'prenom_1', 'last_name': 'nom_1', 'age': 45, 'email': 'adresse_mail_1', 'keyword_preferences': 'horreur'}
Finally, the solution was :
POST /books/user/1/_update
{
"script" : {
"inline": "ctx._source.keyword_preferences += params.preference",
"lang": "painless",
"params" : {
"preference" : ["comique"]
}
}
}

The new function is :
def add_preferences(preferences, i):
doc = es.get(index="books", doc_type="user", id=i)
res = es.update(index="books", doc_type="user", id=i, body={'doc': {'keyword_preferences': [{"preferences": preferences}]}})
The mapping is done and now I've got the result :
{'first_name': 'prenom_1', 'last_name': 'nom_1', 'age': 45, 'email': 'mail_1', 'keyword_preferences': [{'preferences': 'horreur'}]}
So, it has replaced the first preference "comique" by "horreur"

Edited. example answer for your question.
Index a doc
POST /books/user/1
{
"keyword_preferences": ["comique"]
}
Now, update a doc to append horreur in keyword_preferences key.
POST /books/user/1/_update
{
"script": "ctx._source.keyword_preferences += keyword_preferences",
"params": {
"keyword_preferences": ["horreur"]
},
"lang": "groovy"
}
This will update keyword_preferences as ["comique", "horreur"].
If update API throws as exception {"type":"script_exception","reason":"scripts of type [inline], operation [update] and lang [groovy] are disabled"}, then you need to config elasticsearch.yml. Add script.engine.groovy.inline.update: on script.groovy.sandbox.enabled: true in elasticsearch.yml and restart you elasticsearch. I hope this helps.

Related

sqlalchemy select joined table on AsyncSession

So, I'm implementing async functionality on fastAPI using sqlalchemy.ext.asyncio & asyncpg, my problem is that the result of my joined query statement is not eagerly loaded even though I specify to my SQLModel that, that table has __mapper_args__ = {"eager_defaults": True} and used the joinLoaded to the query.
this is my User class as SQLModel
class Users(SQLModel, table=True):
__tablename__: str = "users"
id: Optional[int] = Field(default=None, primary_key=True)
email: str = Field(index=True, sa_column=Column('email', String, unique=True))
password: str = Field(max_length=1024)
username: str = Field(max_length=100, nullable=True, index=True)
...some other fields here...
user_role: Optional[UserRoles] = Relationship(back_populates='user')
user_detail: Optional[UserDetails] = Relationship(back_populates='user')
__mapper_args__ = {"eager_defaults": True}
this is my UserRoles class as SQLModel
class UserRoles(SQLModel, table=True):
__tablename__: str = "user_roles"
id: Optional[int] = Field(default=None, primary_key=True)
role: str
...some other fields here...
user: List["Users"] = Relationship(back_populates='user_role')
this is my fastAPI endpoint implementing sqlalchemy.ext.asyncio
'''
Note:
#user_route.get("/get_all_with_role", response_model=List[UserWithRolesRead])
this will return
response -> 0 -> Users field required (type=value_error.missing)
response -> 0 -> UserRoles field required (type=value_error.missing)
'''
#user_route.get("/get_all_with_role")
async def get_all_with_role(
email=Depends(token_service.secure),
session: AsyncSession = Depends(session_local),
):
async with session as sess:
query = (
select(Users, UserRoles)
.join(UserRoles)
.options(joinedload(Users.user_role))
)
result = await sess.execute(query)
result = result.scalars().first() # I selected only one for the sake of debugging
print(type(result)) # returns <class 'src.models.users.Users'>
print(result)
return result
and the return of print(result) on my console terminal is:
password='qwe123' id=7 active=True created_date=datetime.datetime(2022, 4, 3, 14, 26, 10, 109696)
locked_end_date=None email='admin#admin.com' username='admin' user_role_id=4 last_login_date=None
user_role=UserRoles(id=4, created_date=datetime.datetime(2022, 2, 4, 0, 0), active=True, role='Admin')
as the printed result suggests the user_role has tuple values of UserRoles. But in return result in the last line of #user_route.get("/get_all_with_role") has the value of:
{
"password": "qwe123",
"id": 7,
"active": true,
"created_date": "2022-04-03T14:26:10.109696",
"locked_end_date": null,
"email": "admin#admin.com",
"username": "admin",
"user_role_id": 4,
"last_login_date": null
}
to summarized, the sqlalchemy query statement is properly functioning since, I can get the desired result on my terminal. But then when I try to get the result from the endpoint the the user_role is missing.
[UPDATE]
The result in previous implementation without the async functionality to the endpoint is this:
# fastAPI endpoint not async
#user_route.get("/get_all_with_role")
async def get_all_with_role(email=Depends(token_service.secure)):
_sess = db.session_local()
with _sess as sess:
query = select(Users, UserRoles).join(UserRoles)
result = sess.exec(query)
return result.first()
# Result
{
"Users": {
"password": "$2b$12$ubLo5CMeORXikXrl8gI58OexeUxgqM/HI57yk6briHi1nvmwqO8R.",
"id": 9,
"active": true,
"created_date": "2022-04-05T11:53:13.875607",
"locked_end_date": null,
"email": "0admin",
"username": "0admin",
"user_role_id": 4,
"last_login_date": null
},
"UserRoles": {
"id": 4,
"created_date": "2022-02-04T00:00:00",
"active": true,
"role": "Admin"
}
}

Dynamically call function name in python

I'm trying to call function name dynamically if present in the list of dictionary, if present call the function else exit silently. How can I achieve this, tried below locals approach but didn't works
jobs = [{
"job": "IT",
"company": "google"
},
{
"job": "Sales",
"company": "walmart"
}
]
def IT(name):
print('Full %s' %name )
def Sales(name):
print('View %s' %name)
name = 'department'
Input_Job = 'Sales'
locals()[jobs['job'][Input_Job]](name)
expecting output is Input_Job = 'Sales'
View department
expecting output is Input_Job = 'IT'
Full department

adding new documents not being show in ElasticSearch index

I am new to ElasticsSearch and was messing around with it today. I have a node running on my localhost and was creating/updating my cat index. As I was adding more documents into my cat indexes, I noticed that when I do a GET request to see all of the documents in Postman, the new cats I make are not being added. I started noticing the issue after I added my tenth cat. All code is below.
ElasticSearch Version: 6.4.0
Python Version: 3.7.4
my_cat_mapping = {
"mappings": {
"_doc": {
"properties": {
"breed": { "type": "text" },
"info" : {
"cat" : {"type" : "text"},
"name" : {"type" : "text"},
"age" : {"type" : "integer"},
"amount" : {"type" : "integer"}
},
"created" : {
"type": "date",
"format": "strict_date_optional_time||epoch_millis"
}
}
}
}
}
cat_body = {
"breed" : "Persian Cat",
"info":{
"cat":"Black Cat",
"name": " willy",
"age": 5,
"amount": 1
}
}
def document_add(index_name, doc_type, body, doc_id = None):
"""Funtion to add a document by providing index_name,
document type, document contents as doc and document id."""
resp = es.index(index=index_name, doc_type=doc_type, body=body, id=doc_id)
print(resp)
document_add("cat", "cat_v1", cat_body, 100 )
Since the document id is passed as 100 it just updates the same cat document. I'm assuming its not changed on every run !?
You have to change the document id doc_id with every time to add new cat instead of updating existing ones.
...
cat_id = 100
cat_body = {
"breed" : "Persian Cat",
"info":{
"cat":"Black Cat",
"name": " willy",
"age": 5,
"amount": 1
}
}
...
document_add("cat", "cat_v1", cat_body, cat_id )
With this you can change both cat_id and cat_body to get new cats.

marshmallow EmbeddedDocument doesn't work

I made a simple board api with flask-restplus and mongoengie.
Also use marshmallow for serialize data.
Below code is now I worked.
[model]
class Article(Document):
no = SequenceField()
subject = StringField(required=True)
content = StringField(required=True)
userid = StringField(required=True)
comments = ListField(EmbeddedDocumentField(Comment))
created_at = DateTimeField(default=datetime.datetime.now())
updated_at = DateTimeField(default=datetime.datetime.now())
class Comment(EmbeddedDocument):
content = StringField(required=True)
userid = StringField(required=True)
created_at = DateTimeField(default=datetime.datetime.now())
[serializer]
class CommentSchema(Schema):
content = fields.String()
userid = fields.String()
created_at = fields.DateTime()
class ArticleSchema(Schema):
comments = CommentSchema(many=True)
class Meta:
fields = ('no', 'subject', 'content', 'userid', 'comments', 'created_at', 'updated_at')
I defined schema follow to model.
In ArticleSchema, to show comments, I definded comments = CommentSchema(many=True) and insert it to fields.
And get article function is here.
def get_all_articles():
articles = Article.objects.all()
data, errors = ArticleListSchema(many=True).dump(articles)
return data
But when I access to it, it occur Internal error and throw error message like this.
TypeError: Object of type Comment is not JSON serializable
After searched in google, I found some interest function, Nested. (https://marshmallow.readthedocs.io/en/3.0/nesting.html)
So I modified schema.
class ArticleSchema(Schema):
no = fields.Integer()
subject = fields.String()
content = fields.String()
userid = fields.String()
comments = fields.Nested(CommentSchema())
created_at = fields.DateTime()
updated_at = fields.DateTime()
(comments = fields.Nested(CommentSchema())
But it doesn't work properly too.
[result]
{
"subject": "string",
"content": "string",
"userid": "string",
"updated_at": "2018-11-06T17:04:55.197000+00:00",
"no": 20,
"created_at": "2018-11-06T17:04:55.197000+00:00",
"comments": {}
}
I already insert 2 comments and mongodb result is,
> db.article.find()
{ "_id" : ObjectId("5be14bb61b48d9113e3d1413"), "no" : 20, "subject" : "string", "content" : "string", "userid" : "string", "comments" : [ { "content" : "cosdadas", "userid" : "123123", "created_at" : ISODate("2018-11-06T17:34:44.199Z") }, { "content" : "Second comment", "userid" : "john", "created_at" : ISODate("2018-11-06T17:34:44.199Z") } ], "created_at" : ISODate("2018-11-06T17:04:55.197Z"), "updated_at" : ISODate("2018-11-06T17:04:55.197Z") }
But in API, comments doesn't show. Just empty {}.
Is there any solution here?
Thanks.
[SOLVED]
Change
comments = fields.Nested(CommentSchema()) to
comments = fields.Nested(CommentSchema, many=True) and it works perfectly.

flask-marshmallow custom fields

I use flask-marshmallow and mongoengine.
Also flask-restplus for my API server.
Here is my api.py
class BoardSchema(ma.Schema):
class Meta:
fields = ('no', 'title', 'body', 'tags', 'created_at', 'views')
board_schema = BoardSchema()
boards_schema = BoardSchema(many=True)
class ArticleList(Resource):
def get(self):
articles = Board.objects.all()
return boards_schema.jsonify(articles)
model.py
from datetime import datetime
from mongoengine import *
from config import DB_NAME
connect(DB_NAME)
class Board(Document):
d = datetime.now()
date = "{}-{}-{}".format(d.year, d.month, d.day)
no = SequenceField()
title = StringField(required=True)
body = StringField(required=True)
tags = ListField(StringField())
likes = ListField(StringField())
views = ListField(StringField())
password = StringField(required=True)
created_at = DateTimeField(default=date)
updated_at = DateTimeField(default=date)
When I access to /article, it's result like this ->
{
"body": "123",
"created_at": "2018-08-20T00:00:00+00:00",
"no": 1,
"tags": [
"MySQL",
"C"
],
"title": "\ud14c\uc2a4\ud2b8",
"views": [
"127.0.0.1"
]
}
in "views", ip will be added who read article.
But I want to count of all the list of views and include it to my result.
The result I wanted is here.
{
"body": "123",
"created_at": "2018-08-20T00:00:00+00:00",
"no": 1,
"tags": [
"MySQL",
"C"
],
"title": "\ud14c\uc2a4\ud2b8",
"views": 20
}
I'm new at flask-marshmallow so I'm so confused how can I solve this issue.
Thanks.
Maybe you can try like this:
class BoardSchemaCustom(ma.ModelSchema):
class Meta:
model = Board
views = ma.fields.method(deserialize="_custom_serializer")
def _custom_serializer(self, obj):
return len(obj.views)
Create instance of your custom schema:
custom_board_schema = BoardSchemaCustom()
and dump it:
dump, errors = custom_board_schema.schema.dump(Board.query.first())
>>> dump
i've got the same problem. and my code works after installing marshmallow-sqlalchemy
pip install marshmallow-sqlalchemy
see from offical documentation
https://flask-marshmallow.readthedocs.io/en/latest/
Below snippet would also work:
class BoardSchemaCustom(ma.ModelSchema):
class Meta:
model = Board
views = ma.fields.Function(lambda obj: len(obj.views))

Resources