Im have some data from my collection at mongoDb i want to see all data from specified collection let say i've simple code like this
from pymongo import MongoClient
url = 'my url'
client = MongoClient(url, ssl=True, retryWrites=True)
class DB(object):
def __init__(self):
self.db = client.mydb
self.col = self.db.mycol
def see_listed(self):
for i in self.col.find():
return i
db = DB()
print(db.see_listed())
That only returned one data from my collection
but if i changed code from see_listed to
for i in self.col.find():
print(i)
That return all of data from my collection,where my wrong i don't know.. I just read some documents at try like this.
Im so thankful for any help im appreciate
You only get one document since you use return in your see_listed function.
If you change the return to yield instead it should return a generator you can iterate through.
def see_listed(self):
for i in self.col.find():
yield i
But if you only want the data in a list you could do:
def see_listed(self):
return list(self.col.find())
Maybe not the best choice if the size of the data is unknown.
yield keyword: What does the "yield" keyword do?
Related
I must be doing something wrong
I'm trying to loop through all documents in a collection and add the contents to a dictionary.
I added the data to a dictionary, but my intention was to loop through all docs:
firebase_admin.initialize_app(cred)
db = firestore.client()
blog_col = db.collection(u'blog')
class Art_cont:
def __init__(self,title,date,link):
self.title = title
self.date = date
self.link = link
def getLast4():
query = blog_col.order_by("date").limit_to_last(4).get()
for doc in query: #Need to find a way to loop through all docs, this doesn't work
db_title=doc.to_dict()["title"]
db_date=doc.to_dict()["date"]
db_link=doc.to_dict()["link"]
content1=Art_cont(db_title,db_date,db_link)
#here wrap them up with html and return them to app
print(content1.title,content1.date,content1.link)
When I run that code it only gives me the first doc content:
Vs the other docs that have the same structure:
Any advice would be appreciated.
Assuming that query is an object of your model, write db_title = doc.your_attribute_name.to_dict()['title'] and do the same for the other attributes.
Well, I almost had it, I ended up creating a function instead:
def getlist(x):
query = blog_col.order_by("date").limit_to_last(4).get()
docs=[]
for doc in query:
docs.append(doc)
docnum=docs[x]
db_title=docnum.to_dict()["title"]
db_date=docnum.to_dict()["date"]
db_link=docnum.to_dict()["link"]
content=Art_cont(db_title,db_date,db_link)
return content
print(getlist(0).title)
print(getlist(1).title)
etc...
Hope this helps anyone in a similar situation.
I have the following code structure written in Python3.6, which I need to test using sqlite3 (because of standards defined in my project):
class BigSecretService:
""" Class designed to make calculations based on data stored in MySQL. """
def load_data(self):
# load some data using sqlalchemy ORM
def get_values_from_fields(self, fields):
# here's getting values via sqlalchemy execute with raw query:
self.sql_service.execute(SOME_QUERY)
def process_data(self, data, values):
# again execute some raw query
# process data and put into result list
return reuslt_list
def make_calculations(self, params):
data = self.load_data()
values = self.get_values_from_fields(fields)
result_vector = process_data(data, values)
SOME_QUERY is in separate module and it's format looks like this:
"SELECT SUM(some_field) FROM some_table WHERE col1 = :col1 AND col2 = :col2"
To cover make_calculations in my component test I designed awful patches:
class PatchedConnection:
""" Class is used to transform queries to sqlite format before executing. """
def __init__(self, connection, engine):
self.connection = connection
self.engine = engine
def __call__(self):
conn = self.connection()
conn.execute = self.patched_execute(conn.execute)
return conn
def transform_date(self, date):
try:
# quick check just for testing
if '+00:00' in date:
date = date.replace('T', ' ').replace('+00:00', '.000000')
finally:
return date
def patched_execute(self, f_execute):
def prepare_args_for_sqlite(query, *args):
# check if query is in sqlite format
if args:
if '?' in str(query):
args = list(map(self.transform_date, list(args[0].values())))
return self.engine.execute(str(query), args)
return f_execute(query, args[0])
else:
return f_execute(query)
return prepare_args_for_sqlite
Then in test it looks like this:
QUERY_TEMPLATE_SQLITE = 'SELECT SUM(some_field) FROM some_table WHERE col1 = ? AND col2 = ?'
with mock.patch('path_to_my_service.SOME_QUERY', QUERY_TEMPLATE_SQLITE):
self.sql_service.get_connection = PatchedConnection(self.sql_service.get_connection, self.engine)
response = self.client.simulate_post("/v1/secret_service/make_calculations",
headers=self.auth_header,
body=json.dumps(payload))
self.assertEqual(response.status_code, 200)
# then check response.text
It works so far, but I believe there must be much better solution. Moreover, in patched_execute args from dict are being converted to list, and who knows if order of dict values will be the same all the time.
So, my question is how to perform such testing in a correct way with given tools?
If you need to intercept and manipulate the SQL being sent to the database then using core events https://docs.sqlalchemy.org/en/13/core/events.html would be the most straightforward way of doing this. The before_cursor_execute event would suit your purposes as outlined in the following example from the SQLAlchemy documentation.
#event.listens_for(engine, "before_cursor_execute", retval=True)
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
# do something with statement, parameters
return statement, parameters
From the example you have given however, I'm not sure that this is necessary. The MySQL query you have listed is also a valid SQLite query and needs no manipulation. Also if you pass your parameters as python objects, rather than as strings, then again no manipulation should be needed as SQLAlchemy will map these correctly to the backend.
I have searched some of the questions regarding this topic but i couldn't find a solution to my problem.
I'm currently trying to use multiple parsers on a site depending on the product I want to search. After trying some methods, I ended up with this:
With this start request:
def start_requests(self):
txtfile = open('productosABuscar.txt', 'r')
keywords = txtfile.readlines()
txtfile.close()
for keyword in keywords:
yield Request(self.search_url.format(keyword))
That gets into my normal parse_item.
What i want to do is, with this parse_item (by checking with the item category like laptop, tablet, etc):
def parse_item(self,response):
#I get the items category for the if/else
category = re.sub('Back to search results for |"','', response.xpath('normalize-space(//span[contains(#class, "a-list-item")]//a/text())').extract_first())
#Get the product link, for example (https://www.amazon.com/Lenovo-T430s-Performance-Professional-Refurbished/dp/B07L4FR92R/ref=sr_1_7?s=pc&ie=UTF8&qid=1545829464&sr=1-7&keywords=laptop)
urlProducto = response.request.url
#This can be done in a nicer way, just trying out if it works atm
if category == 'Laptop':
yield response.follow(urlProducto, callback = parse_laptop)
With:
def parse_laptop(self, response):
#Parse things
Any suggestions? The error i get when running this code is 'parse_laptop' is not defined. I have already tried putting the parse_laptop above the parse_item and i still get the same error.
You need to refer to a method and not a function, so just change it like this:
yield response.follow(urlProducto, callback = self.parse_laptop)
yield response.follow(urlProducto, callback = parse_laptop)
This is the request and here's you function def parse_laptop(self, response): you probably have noticed that you parse_laptop function requires self object.
so please modify you request to :
yield response.follow(urlProducto, callback = self.parse_laptop)
This should do the work.
Thanks.
I am using a python package for database managing. The provided class has a method delete() that deletes a record from the database. Before deleting, it asks a user to verify the operation from a console, e.g. Proceed? [yes, No]:
My function needs to perform other actions depending on whether a user chose to delete a record. Can I get user's input requested by the function from the package?
Toy example:
def ModuleFunc():
while True:
a=input('Proceed? [yes, No]:')
if a in ['yes','No']:
#Perform some actions behind a hood
return
This function will wait for one of the two responses and return None once it gets either. After calling this function, can I determine the User's response (without modifying this function)? I think a modification of the Package's source code is not a good idea in general.
Why not just patch the class at runtime? Say you had a file ./lib/db.py defining a class DB like this:
class DB:
def __init__(self):
pass
def confirm(self, msg):
a=input(msg + ' [Y, N]:')
if a == 'Y':
return True
return False
def delete(self):
if self.confirm('Delete?'):
print ('Deleted!')
return
Then in main.py you could do:
from lib.db import DB
def newDelete(self):
if self.confirm('Delete?'):
print('Do some more stuff!')
print('Deleted!')
return
DB.delete = newDelete
test = DB()
test.delete()
See it working here
I would save key events to somewhere(file or memory) with something like Keylogger. Then, you will be able to reuse last one.
However, if you can modify module package 📦 and redistribute, it would be easier.
Return
To
Return a
I got a string from a server response:
responseString:"{"session":"vvSbMInXHRJuZQ==","age":7200,"prid":"901Vjmx9qenYKw","userid":"user_1"}"
then I do:
responseString[1..-2].tokenize(',')
got:
[""session":"vvSbMInXHRJuZQ=="", ""age":7200", ""prid":"901Vjmx9qenYKw"", ""userid":"user_1""]
get(3) got:
""userid":"user_1""
what I need is the user_1, is there anyway I can actually get it? I have been stuck here, other json methods get similar result, how to remove the outside ""?
Thanks.
If you pull out the proper JSON from responseStr, then you can use JsonSlurper, as shown below:
def s = 'responseString:"{"session":"vvSbMInXHRJuZQ==","age":7200,"prid":"901Vjmx9qenYKw","userid":"user_1"}"'
def matcher = (s =~ /responseString:"(.*)"/)
assert matcher.matches()
def responseStr = matcher[0][1]
import groovy.json.JsonSlurper
def jsonSlurper = new JsonSlurper()
def json = jsonSlurper.parseText(responseStr)
assert "user_1" == json.userid
This code can help you get you to the userid.
def str= 'responseString:"{:"session":"vvSbMInXHRJuZQ==","age":7200,"prid":"901Vjmx9qenYKw","userid":"user_1","hdkshfsd":"sdfsdfsdf"}'
def match = (str=~ /"userid":"(.*?)"/)
log.info match[0][1]
this pattern can help you getting any of the values you want from the string. Try replacing userid with age, you will get that
def match = (str=~ /"age":"(.*?)"/)
#Michael code is also correct. Its just that you have clarified that you want the user Name to be specific