I am trying to store the mitmproxy flows to the database for further processing but when using the following code to insert into the sqlite3 database it's just skipping the insert into the database line and no data gets saved nor any errors are showing.
from mitmproxy.options import Options
from mitmproxy.proxy.config import ProxyConfig
from mitmproxy.proxy.server import ProxyServer
from mitmproxy.tools.dump import DumpMaster
import sqlite3
conn = sqlite3.connect('traffic.db')
c = conn.cursor()
c.execute("""CREATE TABLE IF NOT EXISTS traffic (
id INTEGER PRIMARY KEY,
path,
method,
status,
request,
response
)""")
conn.commit()
class Addon(object):
def __init__(self):
self.num = 1
def request(self, flow):
print("flow.request.path")
print(flow.request.path)
c.execute("INSERT INTO traffic VALUES (null, :path, :method, :status, :request, :response)",
{'path': flow.request.path, 'method': flow.request.method, 'status': flow.response.status_code,
'request': flow.request.get_content(), 'response': flow.response.content})
conn.commit()
flow.request.headers["count"] = str(self.num)
def response(self, flow):
self.num = self.num + 1
flow.response.headers["count"] = str(self.num)
print(self.num)
if __name__ == "__main__":
options = Options(listen_host='0.0.0.0', listen_port=8080, http2=True)
m = DumpMaster(options, with_termlog=False, with_dumper=False)
config = ProxyConfig(options)
m.server = ProxyServer(config)
m.addons.add(Addon())
m.run()
What is the correct way of doing this?
I used your code but a different approach, that I used for writing to text file before.
However, this seems to work, so thank you for the SQL part!
import sqlite3
conn = sqlite3.connect('/home/pi/mitm/traffic.db')
c = conn.cursor()
c.execute("""CREATE TABLE IF NOT EXISTS traffic (
id INTEGER PRIMARY KEY,
path,
method,
status,
request,
response
)""")
conn.commit()
def response(flow):
c.execute("INSERT INTO traffic VALUES (null, :path, :method, :status, :request, :response)",
{'path': flow.request.path, 'method': flow.request.method, 'status': flow.response.status_code,
'request': flow.request.content, 'response': flow.response.content})
conn.commit()
The I use mitmdump instead of mitmproxy this way:
mitmdump -s the_code_above.py
This works for me, hopefully it helps for your case, too.
Related
I'm currently writing an API in Flask, and am currently trying to me my POST request to work. Ideally, it would add a band to the DB with a band_ID, name, genre, number of gigs, and a rating. For some reason, running the proper POST request in postman returns a 405. I didn't even know an API *I made * would tell me I don't have access to a POST I'm writing. Do I need to change anything to make it have access?
from flask import Flask, request
from flask_restful import Api, Resource
import sqlite3
app = Flask(__name__)
api = Api(app)
class Band(Resource):
def get(self, band_id):
conn = sqlite3.connect('database.db')
c = conn.cursor()
c.execute("SELECT * FROM bands WHERE band_id=?", (band_id,))
result = c.fetchone()
if result:
return {"message": result}
else:
return {"message": "Band not found"}
def post(self):
data = request.get_json()
if not all(key in data for key in ('band_name', 'band_genre', 'gigs', 'rating')):
return {"message": "Missing data"}
band_name = data['band_name']
band_genre = data['band_genre']
gigs = data['gigs']
rating = data['rating']
conn = sqlite3.connect('database.db')
c = conn.cursor()
c.execute("INSERT INTO bands (band_name,band_genre,gigs,rating) VALUES (?,?,?,?)", (band_name,band_genre,gigs,rating))
conn.commit()
return {"message": "Success"}
api.add_resource(Band, '/bands/<int:band_id>')
if __name__ == "__main__":
conn = sqlite3.connect('database.db')
c = conn.cursor()
c.execute("DROP TABLE IF EXISTS bands")
c.execute("""CREATE TABLE IF NOT EXISTS bands (
band_id INTEGER PRIMARY KEY,
band_name TEXT,
band_genre TEXT,
gigs INTEGER,
rating INTEGER
)""")
conn.commit()
app.run(debug=True)
I've tried changing the call on postman and fiddling around with my add.resource lines but to no avail. Totally lost on what to do next.
from flask import Flask, jsonify, request, url_for, redirect, session, render_template, g
import sqlite3
app = Flask(__name__)
app.config['DATABASE'] = './data.db'
def connect_db():
path = r"C:\Users\Arjun\Documents\flask_app\data.db"
sql = sqlite3.connect(path)
sql.row_factory = sqlite3.Row
return sql
def get_db():
if not hasattr(g, 'sqlite3'):
g.sqlite_db = connect_db()
return g.sqlite_db
#app.teardown_appcontext
def close_db(error):
if hasattr(g, 'sqlite_db'):
g.sqlite_db.close()
#app.route('/home', defaults={'name':'Default'})
#app.route('/home/<name>')
def home(name):
session['name'] = name
return render_template('home.html', name=name, display = False, mylist = [1,2,3,4]
#app.route('/theform5', methods=['GET', 'POST'])
def theform5():
if request.method == 'GET':
return render_template('form.html')
else:
name = request.form['name']
location = request.form['location']
db = get_db()
db.execute("insert into users (name, location) values (?, ?)", [name, location])
db.commit()
return redirect(url_for('home', name=name, location=location))
#app.route('/viewresults')
def viewresults():
db = get_db()
cur = db.execute('select id, name, location from users')
results = cur.fetchall()
return 'The ID is {}. Name is {}. The location is {}. '.format(results[1]['id'], results[1]
['name'], results[1]['location'])
Error in:
return "The ID is {}. Name is {}. The location is {}".format(results[1]['id'],results[1]['name'],results[1]['location'])
IndexError: list index out of range
Though the table can be edited directly through the terminal, it can't be edited when the flask web app is run.
Browser: Chrome
SQLite: v3.30.1
Python: v3.8.1
Flask: v1.1.1
you have a syntax error in your return statement in your home function. missing ')' at the end.
In order to run db.execute you would have to set up the cursor first:
db = get_db()
cursor = db.cursor()
cursor.execute("insert into users (name, location) values (?, ?)", [name, location])
also, maybe your syntax is off, try this:
cursor.execute("insert into users values('%s', '%s')" %(name, location))
Are these not the answers you are looking for?
I have a postgresql function in order to insert values that works fine in psql
CREATE FUNCTION new_msg(p1 type, p2 type)
RETURNS type AS
BEGIN
-- some logic
INSERT INTO table(col1, col2) values (p1,p2);
return value;
END;
LANGUAGE language_name;
A python function like ...
import psycopg2
from config import config
def new_msg(ref_p1, ref_p2):
# Configuracion
params = config()
conn = psycopg2.connect(**params)
cur = conn.cursor()
cur.execute("select * from new_msg(%s,%s);",(ref_p1, ref_p1,))
But when the functions is called in python
new_msg(some_p1,some_p2)
the values are not inserted into the corresponding table and a error is not generated. I also tried callproc method from Psycopg2 is not working. Any suggestion? thanks.
Welcome to StackOverflow!
You need to commit your changes. You can try add cur.commit() after your code to do it every time you write to the DB, or you could try using a context manager to handle it all.
Create a file called mydb.py
import psycopg2
import psycopg2.pool
from contextlib import contextmanager
dbpool = psycopg2.pool.ThreadedConnectionPool(host=<<YourHost>>,
port=<<YourPort>>,
dbname=<<YourDB>>,
user=<<YourUser>>,
password=<<YourPassword>>,
)
#contextmanager
def db_cursor():
conn = dbpool.getconn()
try:
with conn.cursor() as cur:
yield cur
conn.commit()
except:
conn.rollback()
raise
finally:
dbpool.putconn(conn)
Then your code can use:
import mydb
def new_msg(ref_p1, ref_p2):
with mydb.db_cursor as cur:
cur.execute("""
select * from new_msg(%(ref_p1)s,%(ref_p2)s)
""", {
'ref_p1': ref_p1,
'ref_p2': ref_p2,
}
)
return
this code is not inserting my list(self.list2) into the database 'main.db'. I read the the following posts already and they all seem to use the idea of using join() to create # of place holders based on the length of the list.
Dynamically creating a placeholder to insert many column values for a row in SQLite table
Inserting to sqlite dynamically with Python 3
the code is running without errors.
I tested the code by printing
return (f"{', '.join('?' * len(input))}")
and it prints "?, ?, ?, ? ", so I know the function works.
the database is created properly with the following code:
self.cursor.execute('''CREATE TABLE IF NOT EXISTS main
(T_num text Primary Key NOT NULL,
Name text NOT NULL,
Item1 text,
Item2 text,
Item3 text)''')
Maybe I missed a small detail, or I don't know how the return statement/function works.
Please help me to trouble shoot this. Thank you for any assistance.
import tkinter as tk
import sqlite3
class Model():
def __init__(self):
self.list1 = [('Table #', '6'), ('Name', 'Jenn'), ('Beef
Tacos', '6'), ("Fish Tacos", "6")]
self.list2 = list(map(": ".join, self.list1))
self.conn = sqlite3.connect("4th.db")
self.cursor=self.conn.cursor()
self.place_holder(self.list2)
def place_holder(self, input):
return (f"{', '.join('?' * len(input))}")
self.cursor.execute("INSERT INTO main VALUES (place_holder(input))", self.list2)
self.conn.commit()
self.conn.close()
if __name__ == "__main__":
c = Model()
You was trying to insert into db after return in your place_holder method which is not possible because the function exit after return. Also in your sql specify in which column you want to insert into.
like this
self.cursor.execute(f"INSERT INTO main (T_num, Name, Item1, Item2) VALUES {_placeholder}", self.list2)
There is your complete program, hope this will help you.
import tkinter as tk
import sqlite3
class Model():
def __init__(self):
self.list1 = [('Table #', '6'), ('Name', 'Jenn'), ('Beef Tacos', '6'), ("Fish Tacos", "6")]
self.list2 = list(map(": ".join, self.list1))
self.conn = sqlite3.connect("4th.db")
self.cursor = self.conn.cursor()
_placeholder = self.place_holder(self.list2)
# specify here in which column you want to insert the data
self.cursor.execute(f"INSERT INTO main (T_num, Name, Item1, Item2) VALUES {_placeholder}", self.list2)
self.conn.commit()
self.conn.close()
def place_holder(self, input_list):
'''Returns the place holder (?, ?, .....) as string'''
return f"({', '.join('?' * len(input_list))})"
if __name__ == "__main__":
c = Model()
I have a code like this,
# SWAMI KARUPPASWAMI THUNNAI
import jwt
import graphene
from flask import request
from auth.helper import medease_token
from database.get_connection import get_connection
from flask_graphql import GraphQLView
class CredentialInformation(graphene.ObjectType):
"""
graphene object type to get the personal information about the user
"""
country_code = graphene.String()
phone = graphene.String()
verified = graphene.Int()
#medease_token
def resolve_country_code(self, root):
customer_token = request.headers["x-access-token"]
decoded_token = jwt.decode(customer_token, verify=False)
customer_id = decoded_token["customer_id"]
try:
connection = get_connection()
cursor = connection.cursor()
cursor.execute("select country_code from customer_credential where id=%s limit 1", (customer_id, ))
result = cursor.fetchone()
return result["country_code"]
finally:
cursor.close()
connection.close()
#medease_token
def resolve_phone(self, root):
customer_token = request.headers["x-access-token"]
decoded_token = jwt.decode(customer_token, verify=False)
customer_id = decoded_token["customer_id"]
try:
connection = get_connection()
cursor = connection.cursor()
cursor.execute("select phone from customer_credential where id=%s limit 1", (customer_id, ))
result = cursor.fetchone()
return result["phone"]
finally:
cursor.close()
connection.close()
#medease_token
def resolve_verified(self, root):
customer_token = request.headers["x-access-token"]
decoded_token = jwt.decode(customer_token, verify=False)
customer_id = decoded_token["customer_id"]
try:
connection = get_connection()
cursor = connection.cursor()
cursor.execute("select verified from customer_credential where id=%s limit 1", (customer_id,))
result = cursor.fetchone()
return result["verified"]
finally:
cursor.close()
connection.close()
def credential_information_wrapper():
return GraphQLView.as_view("graphql", schema=graphene.Schema(query=CredentialInformation))
which uses flask-graphql and graphene for Python graphql. The code works absolutely fine but I think I am missing something here because I need to open new connections in every resolver, and I need to write same query again so there is a data duplication, so is this the right way of doing or am I missing something?
Any help would be highly appreciated! Thank you in advance.
Opening a new connection on every query (resolver in this case) is okay. You could also set a connection pool to minimize the cost of opening connections.
Just curious, what database adapter or ORM are you using? Would be great if you could refactor the connection and cursor steps into a single function to call on every resolver and only have to pass the SQL query string.