results of sqlite query not displayed in flask web app - python-3.x

I'm attempting to learn flask, so decided to follow this tutorial:
https://www.blog.pythonlibrary.org/2017/12/14/flask-101-adding-editing-and-displaying-data/
I just updated my main function with the below:
#app.route('/results')
def search_results(search):
results = []
search_string = search.data['search']
if search.data['search'] == '':
qry = db_session.query(Album)
results = qry.all()
if not results:
flash('No results found!')
return redirect('/')
else:
# display results
table = Results(results)
table.border = True
return render_template('results.html', table=table)
but when I add an album to the DB and try to query it back using search option it says no results. The DB file was created correctly and I have exactly the same code as in the tutorial up to this point.
The only change I made was adding from tables import Results. Full main.py below. Can you please give me some guidance about where to look for the culprit? Like I said, just learning, so any suggestions re resources in a friendly laid out way would be much appreciated (beginner programmer).
from app import app
from db_setup import init_db, db_session
from forms import MusicSearchForm, AlbumForm
from flask import flash, render_template, request, redirect
from models import Album, Artist
from tables import Results
init_db()
def save_changes(album, form, new=False):
"""
Save the changes to the database
"""
# Get data from form and assign it to the correct attributes
# of the SQLAlchemy table object
artist = Artist()
artist.name = form.artist.data
album.artist = artist
album.title = form.title.data
album.release_date = form.release_date.data
album.publisher = form.publisher.data
album.media_type = form.media_type.data
if new:
# Add the new album to the database
db_session.add(album)
# commit the data to the database
db_session.commit()
#app.route('/', methods=['GET', 'POST'])
def index():
search = MusicSearchForm(request.form)
if request.method == 'POST':
return search_results(search)
return render_template('index.html', form=search)
#app.route('/results')
def search_results(search):
results = []
search_string = search.data['search']
if search.data['search'] == '':
qry = db_session.query(Album)
results = qry.all()
if not results:
flash('No results found!')
return redirect('/')
else:
# display results
table = Results(results)
table.border = True
return render_template('results.html', table=table)
#app.route('/new_album', methods=['GET', 'POST'])
def new_album():
"""
Add a new album
"""
form = AlbumForm(request.form)
if request.method == 'POST' and form.validate():
# save the album
album = Album()
save_changes(album, form, new=True)
flash('Album created successfully!')
return redirect('/')
return render_template('new_album.html', form=form)
if __name__ == '__main__':
app.run()

No doubt you have already peppered your source code with print() statements and found nothing illuminating. Cached rows in the DB model might be the aspect that is hard to understand, here, and logging sqlite calls would shed light on that.
Use this:
import logging
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
It's noisy, but it will show when rows hit the backend DB and when they are retrieved.
Get in the habit of repeatedly issuing debug queries like this, so you know for sure what has been persisted:
$ echo 'select * from album;' | sqlite3 music.db
For repeatable testing, it can be convenient to copy the database file to a backup location, and then cp that frozen snapshot on top of the active file before each test run. It's important that the running flask app be restarted after such copying. Setting FLASK_DEBUG=1 can help with that.
Also, jeverling suggests using SQLAlchemyDebugPanel.

Related

Sorting and Deleting a 1-to-Many relatiionship in SQLALCHEMY and flask

I am working a CRUD flask project that works with a 1-to-many relationship. The end result looks like this, Flask Webform. However when I submit the form for the Update route, it returns to the home page and displays this. Webform After Update.
Here is my DB model
def update(id):
frank = Frankendama.query.get(id)
form = FrankForm()
if form.validate_on_submit():
frank.title=form.title.data
frank.description=form.description.data
frank.tama=form.tama.data
frank.sarado=form.sarado.data
frank.sword=form.sword.data
frank.string=form.string.data
frank.bearing=form.bearing.data
db.session.add(frank)
db.session.commit()
comps = form.companies.data
comps_used = comps.split(",")
all_comps = Company.query.all().filter_by(Company.frankendama_id == id) #Error here
for entry in all_comps:
if entry.frankendama_id == id:
db.session.delete(entry)
for i in range(0, len(comps_used)):
new_entry = comps_used[i]
new_comp = Company(name=new_entry, frankendama_id=id)
db.session.add(new_comp)
db.session.commit()
return redirect(url_for("home"))
else:
return render_template("create.html", form=form)
I am trying to find a query for the Companies table that sorts for all rows with the foreign key 'frankendama_id' that is same to the main tables id. That way i can delete them and then re add them.
When i try using filter() or filter_by() i get the error AttributeError: 'list' object has no attribute 'filter_by'
I am really stumped, any suggestions are welcome! Thanks
move filter_by in front of .all()
all_comps = Company.query.filter_by(frankendama_id = id).all()
edit 1:
Also you can remove 'Company.' when using filter_by and you only need a single equals sign

How to test mysql queries using sqlalchemy and sqlite?

I have the following code structure written in Python3.6, which I need to test using sqlite3 (because of standards defined in my project):
class BigSecretService:
""" Class designed to make calculations based on data stored in MySQL. """
def load_data(self):
# load some data using sqlalchemy ORM
def get_values_from_fields(self, fields):
# here's getting values via sqlalchemy execute with raw query:
self.sql_service.execute(SOME_QUERY)
def process_data(self, data, values):
# again execute some raw query
# process data and put into result list
return reuslt_list
def make_calculations(self, params):
data = self.load_data()
values = self.get_values_from_fields(fields)
result_vector = process_data(data, values)
SOME_QUERY is in separate module and it's format looks like this:
"SELECT SUM(some_field) FROM some_table WHERE col1 = :col1 AND col2 = :col2"
To cover make_calculations in my component test I designed awful patches:
class PatchedConnection:
""" Class is used to transform queries to sqlite format before executing. """
def __init__(self, connection, engine):
self.connection = connection
self.engine = engine
def __call__(self):
conn = self.connection()
conn.execute = self.patched_execute(conn.execute)
return conn
def transform_date(self, date):
try:
# quick check just for testing
if '+00:00' in date:
date = date.replace('T', ' ').replace('+00:00', '.000000')
finally:
return date
def patched_execute(self, f_execute):
def prepare_args_for_sqlite(query, *args):
# check if query is in sqlite format
if args:
if '?' in str(query):
args = list(map(self.transform_date, list(args[0].values())))
return self.engine.execute(str(query), args)
return f_execute(query, args[0])
else:
return f_execute(query)
return prepare_args_for_sqlite
Then in test it looks like this:
QUERY_TEMPLATE_SQLITE = 'SELECT SUM(some_field) FROM some_table WHERE col1 = ? AND col2 = ?'
with mock.patch('path_to_my_service.SOME_QUERY', QUERY_TEMPLATE_SQLITE):
self.sql_service.get_connection = PatchedConnection(self.sql_service.get_connection, self.engine)
response = self.client.simulate_post("/v1/secret_service/make_calculations",
headers=self.auth_header,
body=json.dumps(payload))
self.assertEqual(response.status_code, 200)
# then check response.text
It works so far, but I believe there must be much better solution. Moreover, in patched_execute args from dict are being converted to list, and who knows if order of dict values will be the same all the time.
So, my question is how to perform such testing in a correct way with given tools?
If you need to intercept and manipulate the SQL being sent to the database then using core events https://docs.sqlalchemy.org/en/13/core/events.html would be the most straightforward way of doing this. The before_cursor_execute event would suit your purposes as outlined in the following example from the SQLAlchemy documentation.
#event.listens_for(engine, "before_cursor_execute", retval=True)
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
# do something with statement, parameters
return statement, parameters
From the example you have given however, I'm not sure that this is necessary. The MySQL query you have listed is also a valid SQLite query and needs no manipulation. Also if you pass your parameters as python objects, rather than as strings, then again no manipulation should be needed as SQLAlchemy will map these correctly to the backend.

Unable to insert data into table using python and oracle database

My database table looks like this
I have a web crawler that fetches news from the website and i am trying to store it in this table. I have used scrappy and beautiful soup libraries.The below code shows my crawler logic.
import requests
from bs4 import BeautifulSoup
import os
import datetime
import cx_Oracle
def scrappy(url):
try:
r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')
title = soup.find('title').text.split('|')[0]
time =soup.find('span', attrs={'class':'time_cptn'}).find_all('span')[2].contents[0]
full_text =soup.find('div', attrs={'class':'article_content'}).text.replace('Download The Times of India News App for Latest India News','')
except:
return ('','','','')
else:
return (title,time,url,full_text)
def pathmaker(name):
path = "Desktop/Web_Crawler/CRAWLED_DATA/{}".format(name)
try:
os.makedirs(path)
except OSError:
pass
else:
pass
def filemaker(folder,links_all):
#k=1
for link in links_all:
scrapped=scrappy(link)
#textfile=open('Desktop/Web_Crawler/CRAWLED_DATA/{}/text{}.txt'.format(x,k),'w+')
#k+=1
Title = scrapped[0]
Link = scrapped[2]
Dates = scrapped[1]
Text = scrapped[3]
con = cx_Oracle.connect('shivams/tiger#127.0.0.1/XE')
cursor = con.cursor()
sql_query = "insert into newsdata values(:1,:2,:3,:4)"
cursor.executemany(sql_query,[Title,Link,Dates,Text])
con.commit()
cursor.close()
con.close()
#textfile.write('Title\n{}\n\nLink\n{}\n\nDate & Time\n{}\n\nText\n{}'.format(scrapped[0],scrapped[2],scrapped[1],scrapped[3]))
#textfile.close()
con.close()
folders_links=[('India','https://timesofindia.indiatimes.com/india'),('World','https://timesofindia.indiatimes.com/world'),('Business','https://timesofindia.indiatimes.com/business'),('Homepage','https://timesofindia.indiatimes.com/')]
for x,y in folders_links:
pathmaker(x)
r = requests.get(y)
soup = BeautifulSoup(r.text, 'html.parser')
if x!='Homepage':
links =soup.find('div', attrs={'class':'main-content'}).find_all('span', attrs={'class':'twtr'})
links_all=['https://timesofindia.indiatimes.com'+links[x]['data-url'].split('?')[0] for x in range(len(links))]
else:
links =soup.find('div', attrs={'class':'wrapper clearfix'})
total_links = links.find_all('a')
links_all=[]
for p in range(len(total_links)):
if 'href' in str(total_links[p]) and '.cms' in total_links[p]['href'] and 'http' not in total_links[p]['href'] and 'articleshow' in total_links[p]['href'] :
links_all+=['https://timesofindia.indiatimes.com'+total_links[p]['href']]
filemaker(x,links_all)
Earlier i was creating text files and storing the news in them but now i want to store it in the database for my web application to access it. My database logic is in the file maker function. I am trying to insert the values in the table but its not working and giving various types of errors. I followed other posts on the website but they didnt work in my case. Can anyone help me with this. Also i am not sure if that is the correct way to insert CLOB data as i am using it for the first time. Need help.
You can do the following:
cursor.execute(sql_query, [Title, Link, Dates, Text])
or, if you build up a list of these values, you can then do the following:
allValues = []
allValues.append([Title, Link, Dates, Text])
cursor.executemany(sql_query, allValues)
Hope that explains things!

Tweepy exception when storing trends in a database not when printing them

I am using Python 3.6.2 and Tweepy 3.3.0.
So when I try to retrieve trends from Twitter via the Tweepy API and just print them in my console it works pretty fine like this:
auth = tweepy.OAuthHandler(cons_tok, cons_sec)
auth.set_access_token(app_tok, app_sec)
twitter_api = tweepy.API(auth)
trends = twitter_api.trends_place(1)
for trend in trends[0]["trends"]:
print(trend['name'])
It works fine and I have a list.
Now when I try to store these into a database (I choose SQLite just to try):
class TwitterMain():
def __init__(self, conn):
self.auth = tweepy.OAuthHandler(cons_tok, cons_sec)
self.auth.set_access_token(app_tok, app_sec)
self.api = tweepy.API(self.auth)
self.conn = conn
self.c = self.conn.cursor()
def get_trends(self):
trends = self.api.trends_place(1)
trend_data = []
for trend in trends[0]["trends"]:
trend_tweets = []
trend_tweets.append(trend['name'])
tt = tweepy.Cursor(self.api.search, q = trend['name']).items(3)
for t in tt:
trend_tweets.append(self.get_tweet_html(t.id))
trend_data.append(tuple(trend_tweets))
self.c.executemany("INSERT INTO trend_data VALUES (?,?,?,?, DATETIME('now'))", trend_data)
self.conn.commit()
def get_tweet_html(self, id):
oembed = self.api.get_oembed(id=id, hide_media = True, hide_thread = True)
tweet_html = oembed['html'].strip("\n")
return tweet_html
if __name__ == "__main__":
try:
conn = sqlite3.connect(db)
twit = TwitterMain(conn)
twit.get_trends()
except Exception as e:
print(e.__doc__)
finally:
conn.close()
This is just the part where I store the trends data. The database is already created as well as the table.
Running this creates a Tweepy Exception, and when searching online it says it's the time limit but if so why when I just print, it works fine and not when I try to store them?
There is no table to insert the value into, and the code that you have provided is throwing an exception:
self.c.executemany("INSERT INTO trend_data VALUES (?,?,?,?, DATETIME('now'))", trend_data)
sqlite3.OperationalError: no such table: trend_data
Just add a line to create the table as well in the init method:
self.c.execute("""CREATE TABLE mytable
(c1,c2,c3,c4,c5)""")
and use this to fill in the values:
self.c.executemany("INSERT INTO mytable VALUES (?,?,?,?, DATETIME('now'))", trend_data)

I cannot save rows to the DB with SQLAchemy + Pyramid

After hours of debugging, and because my organization does not have a lot of Python expertise, I am turning to this community for help.
I am trying to follow this tutorial with the goal of committing some data to the database. Although no errors get reported, I am also not saving any rows. What am I doing wrong?
When trying to commit using the db2Session, I get:
Transaction must be committed using the transaction manager.
But nowhere in the tutorial, do I see the transaction manager being used. I thought that this manager is bound using zope.sqlalchemy? Yet, nothing is happening otherwise. Help again would be really appreciated!
I have the following setup in my main function in a Pyramid App:
from sqlalchemy import engine_from_config
from .models import db1Session, db2Session
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
db1_engine = engine_from_config(settings, 'db1.')
db2_engine = engine_from_config(settings, 'db2.')
db1Session.configure(bind=db1_engine)
db2Session.configure(bind=db2_engine)
In .models/__init__py, I have:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import (scoped_session, sessionmaker)
from zope.sqlalchemy import ZopeTransactionExtension
db1Session = scoped_session(sessionmaker(
extension=ZopeTransactionExtension()))
db2Session =
scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
Base = declarative_base()
In ./model/db2.py I have:
class PlateWellResult(Base):
__tablename__ = 'SomeTable'
__table_args__ = {"schema": 'some_schema'}
id = Column("ID", Integer, primary_key=True)
plate_id = Column("PlateID", Integer)
hit_group_id = Column("HitID", Integer, ForeignKey(
'some_schema.HitGroupID.ID'))
well_loc = Column("WellLocation", String)
The relevant bits of my saving function look like this. ./lib/db2_api.py:
def save_selected_rows(input_data, selected_rows, hit_group_id):
""" Wrapper method for saving selected rows """
# Assume I have all the right data below.
new_hit_row = PlateWellResult(
plate_id=master_plate_id,
hit_group_id=hit_group_id,
well_loc=selected_df_row.masterWellLocation)
db1Session.add(new_hit_row)
# When I try the row below:
# db2Session.commit()
# I get: Transaction must be committed using the transaction manager
# If I cancel the line above, nothing gets committed.
return 'Save successful.'
That function is called from my viewer:
#view_config(route_name='some_routename', renderer='json',
permission='create_hit_group')
def save_to_hitgroup(self):
""" Backend to AJAX call to save selected rows to a hit_group """
try:
# Assume that all values were checked and all the right
# parameters are passed
status = save_selected_rows(some_result,
selected_rows_list,
hitgroup_id)
json_resp = json.dumps({'errors': [],
'status': status})
return json_resp
except Exception as e:
json_resp = json.dumps({'errors': ['Error during saving. {'
'0}'.format(e)],
'status': []})
return json_resp
The comments above are good. I just wanted to summarize here.
The transaction manager is begun/committed/aborted by pyramid_tm. If you aren't using that then it's likely the issue.
You are also squashing possible database exceptions which need to be conveyed to the transaction manager. You can do this via transaction.abort() in the exception handler.

Resources