I am using cherrypy 3.6.0, nginx 1.6.2 and uWSGI 2.0.9 on amazon linux.
uwsgi.json config:
{
"uwsgi": {
"socket": ["127.0.0.1:8080"],
"master": True,
"processes": 1,
"threads": 24,
"uid": "ec2-user",
"protocol": "http",
"vacuum": True
"chdir": "/usr/local/src/myapp",
"wsgi-file": "/usr/local/src/myapp/wsgi.py",
"logto": "/var/log/uwsgi.log"
}
}
I try enabling sessions:
cherrypy.config.update({
'server.socket_host': '0.0.0.0',
'engine.autoreload.on': False,
'server.socket_port': 8080,
'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
'tools.sessions.on' : True,
'tools.sessions.storage_type': "file",
'tools.sessions.storage_path': "/home/ec2-user/.cherrypy_sessions",
'tools.sessions.timeout': 60
})
to use them here:
import cherrypy
import random
import string
import json
import urllib.parse
class IdentityApi(object):
#cherrypy.expose
def gen_crosssite(self):
crsf = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(32))
cherrypy.session['my_state'] = crsf;
return json.dumps({'crsf': crsf})
I get the following exception:
[25/Feb/2015:15:51:36] HTTP Traceback (most recent call last):
File "/usr/local/lib/python3.4/site-packages/cherrypy/_cprequest.py", line 670, in respond
response.body = self.handler()
File "/usr/local/lib/python3.4/site-packages/cherrypy/lib/encoding.py", line 217, in __call__
self.body = self.oldhandler(*args, **kwargs)
File "/usr/local/lib/python3.4/site-packages/cherrypy/_cpdispatch.py", line 61, in __call__
return self.callable(*self.args, **self.kwargs)
File "./api/IdentityApi.py", line 25, in gen_crsf
cherrypy.session['my_state'] = crsf;
AttributeError: 'module' object has no attribute 'session'
update: Running it from the commandline 'python3 wsgi.py' and WITHOUT uWSGI works.
Had to comment out:
cherrypy.server.unsubscribe()
cherrypy.engine.start()
updated wsgi.py:
if __name__ == '__main__':
cherrypy.log('jobtagr log file = %s', cherrypy.log.access_file)
setup_logging()
cherrypy.config.update({
'server.socket_host': '0.0.0.0',
'engine.autoreload.on': False,
'server.socket_port': 8080,
'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
'tools.sessions.on' : True,
'tools.sessions.storage_type': "file",
'tools.sessions.storage_path': "/home/ec2-user/.cherrypy_sessions",
'tools.sessions.timeout': 60
})
# cherrypy.server.unsubscribe()
# cherrypy.engine.start()
cherrypy.quickstart(application)
How do I get sessions to work under uWSGI?
Observations:
cherrypy.session attribute doesn't exist unless sessions.init is called by the session tool,
Don't mix global (server, engine) and application (tools, dispatch) configs,
Don't mess with routing. You've declared HTTP verb dispatch, but I see non-verb gen_crosssite.
Update
I've modelled your case and see the problem now. It's documented behaviour but it is a little subtle anyway. It happens when you switch from cherrypy.quickstart which is more demo thing, to managing real setup with cherrypy.tree.mount without paying attention to what happens with configuration. Here's cherrypy.quickstart, it's small:
def quickstart(root=None, script_name="", config=None):
if config:
_global_conf_alias.update(config)
tree.mount(root, script_name, config)
engine.signals.subscribe()
engine.start()
engine.block()
As you see, it sets both, server (global) config and application config in tree.mount. Here's the warning from Basics documentation section about global config:
cherrypy.config.update() is not meant to be used to configure the application. It is a common mistake. It is used to configure the server and engine.
And here's Combined Configuration Files section:
If you are only deploying a single application, you can make a single config file that contains both global and app entries. Just stick the global entries into a config section named [global] (or top level key global), and pass the same file to both config.update and tree.mount. If you’re calling cherrypy.quickstart(app root, script name, config), it will pass the config to both places for you. But as soon as you decide to add another application to the same site, you need to separate the two config files/dicts.
Here goes your case modelled with two applications:
#!/usr/bin/env python3
import random
import string
import cherrypy
class Api:
def __init__(self):
self.string = String()
# ...
class String:
exposed = True
def GET(self):
return cherrypy.session.get('mystring', '')
def POST(self, length=8):
result = ''.join(random.sample(string.hexdigits, int(length)))
cherrypy.session['mystring'] = result
return result
def PUT(self, new):
cherrypy.session['mystring'] = new
def DELETE(self):
cherrypy.session.pop('mystring', None)
class Token:
#cherrypy.expose
def csrf(self):
crsf = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(32))
cherrypy.session['csrf'] = crsf
return {'crsf': crsf}
if __name__ == '__main__':
# global server config
cherrypy.config.update({
'server.socket_host' : '0.0.0.0',
'server.socket_port' : 8080,
})
# applicaton config is provided
cherrypy.tree.mount(Api(), '/api', {
'/' : {
'tools.sessions.on' : True,
'request.dispatch' : cherrypy.dispatch.MethodDispatcher(),
},
'/string' : {
'tools.response_headers.on' : True,
'tools.response_headers.headers' : [('Content-Type', 'text/plain')]
}
})
# applicaton config is provided
cherrypy.tree.mount(Token(), '/token', {'/' : {
'tools.sessions.on' : True,
'tools.json_out.on' : True,
}})
cherrypy.engine.signals.subscribe()
cherrypy.engine.start()
cherrypy.engine.block()
I got this error very recently, it turned out to be a problem with my uwsgi configuration. I suggest checking your uwsgi error log and pasting the last few lines here.
Here's my config file; perhaps compare it to yours and make sure you have correct values for each.
chdir=/root/Dropbox/QUTE_deployment
module=QUTE:app
master=True
vacuum=True
socket=/tmp/qute.sock
pidfile=/tmp/qute.pid
processes=1
threads=24
daemonize=/var/log/uwsgi.log
wsgi-file=/root/Dropbox/QUTE_deployment/QUTE.py
Related
I am trying to make a database in my terminal but when I try it I always get this error when I use from first import db:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ModuleNotFoundError: No module named 'first'
Here is my code
from flask import Flask, render_template
from flask_sqlalchemy import SQLAlchemy
from datetime import datetime
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///posts.db'
db = SQLAlchemy(app)
class BlogPost(db.Model):
id = db.Column(db.Integer, primary_key= True)
Title = db.Column(db.String(100), nullable= False)
Content = db.Column(db.Text, nullable= False)
Date_posted = db.Column(db.DateTime, nullable= False, default=datetime.utcnow)
def __repr__(self):
return "Blog Post" + str(self.id)
all_posts = [
{
'Title': 'Post 1',
'Content': "This is the first content"
},
{
'Title': 'Post 2',
'Content': 'This is the second content'
}
]
#app.route('/')
def index():
return render_template("index.html")
#app.route('/blog-posts')
def blog_posts():
return render_template("posts.html", posts = all_posts)
#app.route('/home')
def hello():
return "My first python website"
if __name__ == '__main__':
app.run(debug= True)
I only have python 3.8 installed.
What could be the problem? I've searched the internet and used every advice on other platforms yet no solution. Flask and flask-sqlalchemy are installed properly.
This is the youtube video I use as a guide when creating this project:
https://www.youtube.com/watch?v=3mwFC4SHY-Y
Well I am assuming you have created a python file named first. I am listing out what could have been gone wrong:
You are in the wrong directory.
You have not installed the necessary package. (unlikely)
According to the tutorial, the file name is app.py so there he performs from app import db. Just make sure you have the name of the file correct.
I am not able to think of any other probable cause, unless first is a package create by you to store the project's python files like routes.py, models.py, etc. In that case you need to have __init__.py file in the package directory and initialize app and db in __init__.py file
Is a way to use scrapy splash without docker. I mean, I have a server running with python3 without docker installed. And If possible I don't want to install docker on it.
Also, what does exactly SPLASH_URL. Can I use only the IP of my server ?
I already tried something :
def start_requests(self):
url = ["europages.fr/entreprises/France/pg-20/resultats.html?ih=01510;01505;01515;01525;01530;01570;01565;01750;01590;01595;01575;01900;01920;01520;01905;01585;01685;01526;01607;01532;01580;01915;02731;01700;01600;01597;01910;01906"]
print(url)
yield SplashRequest(url = 'https://' + url[0], callback = self.parse_all_links,
args={
# optional; parameters passed to Splash HTTP API
'wait': 0.5,
# 'url' is prefilled from request url
# 'http_method' is set to 'POST' for POST requests
# 'body' is set to request body for POST requests
} # optional; default is render.html
) ## TO DO : Changer la callback
with setting.py
SPIDER_MIDDLEWARES = {
'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
}
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'
# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
#'Europages.middlewares.EuropagesDownloaderMiddleware': 543,
'scrapy_splash.SplashCookiesMiddleware': 723,
'scrapy_splash.SplashMiddleware': 725,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}
AND
SPLASH_URL = "url_of_my_server"
I hope my post is clear.
Thanks
Regards,
It seems that it used to be possible in previous versions of Splash but not anymore (https://splash.readthedocs.io/en/3.3.1/install.html)
I want to be able to get a body of the specific subrequest using a selenium behind the proxy.
Now I'm using python + selenium + chromedriver. With logging I'm able to get each subrequest's headers but not body. My logging settings:
caps['loggingPrefs'] =
{'performance': 'ALL',
'browser': 'ALL'}
caps['perfLoggingPrefs'] = {"enableNetwork": True,
"enablePage": True,
"enableTimeline": True}
I know there are several options to form a HAR with selenium:
Use geckodriver and har-export-trigger. I tried to make it work with the following code:
window.foo = HAR.triggerExport().then(harLog => { return(harLog); });
return window.foo;
Unfortunately, I don't see the body of the response in the returning data.
Use browsermob proxy. The solution seems totally fine but I didn't find the way to make browsermob proxy work behind the proxy.
So the question is: how can I get the body of the specific network response on the request made during the downloading of the webpage with selenium AND use proxies.
UPD: Actually, with har-export-trigger I get the response bodies, but not all of them: the response body I need is in json, it's MIME type is 'text/html; charset=utf-8' and it is missing from the HAR file I generate, so the solution is still missing.
UPD2: After further investigation, I realized that a response body is missing even on my desktop firefox when the har-export-trigger add-on is turned on, so this solution may be a dead-end (issue on Github)
UPD3: This bug can be seen only with the latest version of har-export-trigger. With version 0.6.0. everything works just fine.
So, for future googlers: you may use har-export-trigger v. 0.6.0. or the approach from the accepted answer.
I have actually just finished to implemented a selenium HAR script with tools you are mentioned in the question. Both HAR getting from har-export-trigger and BrowserMob are verified with Google HAR Analyser.
A class using selenium, gecko driver and har-export-trigger:
class MyWebDriver(object):
# a inner class to implement custom wait
class PageIsLoaded(object):
def __call__(self, driver):
state = driver.execute_script('return document.readyState;')
MyWebDriver._LOGGER.debug("checking document state: " + state)
return state == "complete"
_FIREFOX_DRIVER = "geckodriver"
# load HAR_EXPORT_TRIGGER extension
_HAR_TRIGGER_EXT_PATH = os.path.abspath(
"har_export_trigger-0.6.1-an+fx_orig.xpi")
_PROFILE = webdriver.FirefoxProfile()
_PROFILE.set_preference("devtools.toolbox.selectedTool", "netmonitor")
_CAP = DesiredCapabilities().FIREFOX
_OPTIONS = FirefoxOptions()
# add runtime argument to run with devtools opened
_OPTIONS.add_argument("-devtools")
_LOGGER = my_logger.get_custom_logger(os.path.basename(__file__))
def __init__(self, log_body=False):
self.browser = None
self.log_body = log_body
# return the webdriver instance
def get_instance(self):
if self.browser is None:
self.browser = webdriver.Firefox(capabilities=
MyWebDriver._CAP,
executable_path=
MyWebDriver._FIREFOX_DRIVER,
firefox_options=
MyWebDriver._OPTIONS,
firefox_profile=
MyWebDriver._PROFILE)
self.browser.install_addon(MyWebDriver._HAR_TRIGGER_EXT_PATH,
temporary=True)
MyWebDriver._LOGGER.info("Web Driver initialized.")
return self.browser
def get_har(self):
# JSON.stringify has to be called to return as a string
har_harvest = "myString = HAR.triggerExport().then(" \
"harLog => {return JSON.stringify(harLog);});" \
"return myString;"
har_dict = dict()
har_dict['log'] = json.loads(self.browser.execute_script(har_harvest))
# remove content body
if self.log_body is False:
for entry in har_dict['log']['entries']:
temp_dict = entry['response']['content']
try:
temp_dict.pop("text")
except KeyError:
pass
return har_dict
def quit(self):
self.browser.quit()
MyWebDriver._LOGGER.warning("Web Driver closed.")
A subclass adding BrowserMob proxy for your reference as well:
class MyWebDriverWithProxy(MyWebDriver):
_PROXY_EXECUTABLE = os.path.join(os.getcwd(), "venv", "lib",
"browsermob-proxy-2.1.4", "bin",
"browsermob-proxy")
def __init__(self, url, log_body=False):
super().__init__(log_body=log_body)
self.server = Server(MyWebDriverWithProxy._PROXY_EXECUTABLE)
self.server.start()
self.proxy = self.server.create_proxy()
self.proxy.new_har(url,
options={'captureHeaders': True,
'captureContent': self.log_body})
super()._LOGGER.info("BrowserMob server started")
super()._PROFILE.set_proxy(self.proxy.selenium_proxy())
def get_har(self):
return self.proxy.har
def quit(self):
self.browser.quit()
self.proxy.close()
MyWebDriver._LOGGER.info("BroswerMob server and Web Driver closed.")
I'm currently writing a simple flask application and want to test that my configuration options are working.
Here's my config.py:
import logging
import logging.handlers
import os
class BaseConfig(object):
DEBUG = False
TESTING = False
...
class DevelopmentConfig(BaseConfig):
DEBUG = True
TESTING = True
...
class TestingConfig(BaseConfig):
DEBUG = False
TESTING = True
...
config = {
'development': 'app.config.DevelopmentConfig',
'testing': 'app.config.TestingConfig',
'default': 'app.config.BaseConfig'
}
def configure_app(app):
config_name = os.environ.get('FLASK_CONFIGURATION', 'default')
app.config.from_object(config[config_name])
app.config.from_pyfile('config.py', silent=True)
apikey = os.environ.get('TOKEN', None)
if apikey:
app.config['APIKEY'] = apikey
I call configure_app in my __init__.py right after I create the Flack object.
app = Flask(__name__)
configure_app(app)
config.py:
APIKEY = 'filesecret'
HOOKS = {'testpush': 'scripts/test.sh'}
What I want to do is be able to unittest using py.test the various configuration options. My current attempt is to try and use mock_open, but the configuration keep reading what's in the file instead of skipping it.
class TestConfig:
def setup(self):
self.app = app.app
self.app.config['TESTING'] = True
def test_default(self, mocker):
m = mocker.mock_open(read_data='')
configure_app(self.app)
assert self.app.config['APIKEY'] == "secret"
results of test:
def test_default(self, mocker):
m = mocker.mock_open(read_data='')
configure_app(self.app)
> assert self.app.config['APIKEY'] == "secret"
E assert 'filesecret' == 'secret'
I'm not sure how to do this when I don't know the internals of flask. Is there a way to fake/mock the file? I want to be able to write the various test cases of the configuration file being present, not present, and the environment variable being set to make sure priority is maintained.
So I was going about this all wrong. I looked at Flask's source code and saw how they were testing their configuration functions, and they simply instantiated a new Flask object with their own environment variables.
Flask project test_config.py
Using the context manager found on this stackoverflow question (currently the bottom answer with contextlib) to set my environment variables I got this all working great.
Final test:
#pytest.mark.parametrize(
'envtoken, flask_conf, test_path, apikey, debug, testing', [
('', 'default', 'missing', 'secret', False, False),
('', 'development', 'missing', 'secret', True, True),
('', 'testing', 'missing', 'secret', False, True),
('', 'default', '../tests/data', 'filesecret', False, False),
('envsecret', 'default', '../tests/data/', 'envsecret', False, False)])
def test_config_app(envtoken, flask_conf, test_path, apikey, debug, testing):
with set_env(DOCKERHOOK_TOKEN=envtoken, FLASK_CONFIGURATION=flask_conf):
# app = Flask(__name__)
base_path = get_app_base_path()
instance_path = os.path.join(base_path, test_path)
app = Flask(__name__,
instance_path=instance_path,
instance_relative_config=True)
configure_app(app)
assert app.config['APIKEY'] == apikey
assert app.config['DEBUG'] == debug
assert app.config['TESTING'] == testing
I have a cherrypy application like this:
import cherrypy
from controllers import UsersController
class Root(object):
exposed = True
def index(self):
return 'welcome'
if __name__ == '__main__':
root = Root()
root.users = UsersController()
cherrypy.tree.mount(
root,
'/',
{
'/users' : {'request.dispatch' : cherrypy.dispatch.MethodDispatcher()}
}
)
cherrypy.engine.start()
cherrypy.engine.block()
Now I wish to use MethodDispatcher() for providing REST api to /users resource and I want a default dispatcher for '/' path (where a call to root.index() is expected). Instead of writing own RoutesDispatcher() is there any way to achieve this? (e.g. using MethodDispatcher() for '/users' as shown and something like DefaultDispatcher() for '/')? BTW, the error I am getting is 'Root' object is not callable.
Since your Root is to be served with a normal dispatcher, it should be index.exposed = True.