I'm trying to write my own little python flask app to monitor the hard drives of my server.
But since now, I'm getting trouble using the sqljobstore of apscheduler.
While the server is running, everything is fine. But after a restart, I can't access the web interface and getting the folowing output:
Unable to restore job "refresh_disks" -- removing it
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/apscheduler/util.py", line 289, in ref_to_obj
obj = getattr(obj, name)
AttributeError: module 'dirkules.tasks' has no attribute 'refresh_disks'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/apscheduler/jobstores/sqlalchemy.py", line 141, in _get_jobs
jobs.append(self._reconstitute_job(row.job_state))
File "/usr/local/lib/python3.6/dist-packages/apscheduler/jobstores/sqlalchemy.py", line 128, in _reconstitute_job
job.__setstate__(job_state)
File "/usr/local/lib/python3.6/dist-packages/apscheduler/job.py", line 272, in __setstate__
self.func = ref_to_obj(self.func_ref)
File "/usr/local/lib/python3.6/dist-packages/apscheduler/util.py", line 292, in ref_to_obj
raise LookupError('Error resolving reference %s: error looking up object' % ref)
LookupError: Error resolving reference dirkules.tasks:refresh_disks: error looking up object
[2019-04-26 15:46:39 +0200] [13296] [INFO] Shutting down: Master
Here is my config.py:
import os
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
#from apscheduler.jobstores.memory import MemoryJobStore
baseDir = os.path.abspath(os.path.dirname(__file__))
SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(baseDir, 'dirkules.db')
SQLALCHEMY_TRACK_MODIFICATIONS = False
# The SCHEDULER_JOB_DEFAULTS configuration is per job, that means each job can execute at most 3 threads at the same time.
# The SCHEDULER_EXECUTORS is a global configuration, in this case, only 1 thread will be used for all the jobs.
# I believe the best way for you is to use max_workers: 1 when running locally
SCHEDULER_JOBSTORES = {'default': SQLAlchemyJobStore(url='sqlite:///' + os.path.join(baseDir, 'dirkules.db'))}
#SCHEDULER_JOBSTORES = {'default': MemoryJobStore()}
SCHEDULER_EXECUTORS = {'default': {'type': 'threadpool', 'max_workers': 3}}
SCHEDULER_JOB_DEFAULTS = {'coalesce': False, 'max_instances': 1}
SCHEDULER_API_ENABLED = True
init.py:
import dirkules.config as config
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from flask_apscheduler import APScheduler
app = Flask(__name__)
app.config.from_object(config)
db = SQLAlchemy(app)
import dirkules.models
db.create_all()
scheduler = APScheduler()
scheduler.init_app(app)
scheduler.start()
##app.before_first_request
from dirkules import tasks
# from dirkules.models import Time
# from sqlalchemy.orm.exc import NoResultFound
#
# try:
# Time.query.one()
# except NoResultFound:
# db.session.add(Time("Drives"))
# db.session.commit()
import dirkules.views
and tasks.py:
from dirkules import scheduler
import datetime
import dirkules.driveManagement.driveController as drico
#scheduler.task('interval', id='refresh_disks', seconds=10)
def refresh_disks():
#drives = drico.getAllDrives()
print("Drives refreshed")
Hopefully, you can help me!
Starting the scheduler as a side-effect of importing the module is considered bad practice and is also the likely reason why the attribute lookup fails. I would have to see a simplified, more complete example that reproduces the problem to be sure, however.
Related
My workflow is running perfect on Experimentation, but after deployed to web service, I receive this error while post.
Python Code:
# -*- coding: utf-8 -*-
#import sys
import pickle
import pandas as pd
from sklearn.tree import DecisionTreeClassifier
from sklearn import tree
def azureml_main(dataframe1 = None, dataframe2 = None):
print('input dataframe1 ',dataframe1)
decision_tree_pkl_predictive_maint = r'.\Script Bundle\decision_tree_pkl_predictive_maint.pkl'
#sys.path.insert(0,".\Script Bundle")
#model = pickle.load(open(".\Script Bundle\decision_tree_pkl_predictive_maint.pkl", 'rb'))
modle_file = open(decision_tree_pkl_predictive_maint,"rb")
model = pickle.load(modle_file)
#return the mode of prediciton
result = model.predict(dataframe1)
print(result)
result_df = pd.DataFrame({'prediction_class':result})
return result_df,
ERROR:
Execute Python Script RRS : Error 0085: The following error occurred during script evaluation, please view the output log for more information: ---------- Start of error message from Python interpreter ---------- Caught exception while executing function: Traceback (most recent call last): File "\server\InvokePy.py", line 120, in executeScript outframe = mod.azureml_main(*inframes) File "\temp-1036260731852293620.py", line 46, in azureml_main modle_file = open(decision_tree_pkl_predictive_maint,"rb") FileNotFoundError: [Errno 2] No such file or directory: '.\Script Bundle\decision_tree_pkl_predictive_maint.pkl' ---------- End of error message from Python interpreter ----------
Please, Advice.
The issue has to do with your file path. Ensure that you have included the correct path.
I try use instead time.sleep schedule functions but I have error
package containing module schedule is not listed in project
requirements
in file with error it's look like
supervisor: couldn't exec
/home/darek/PycharmProjects/Small_programs/shell_scripts/test/test_s.py:
ENOEXEC supervisor: child process was not spawned
when I use code:
import requests, time, schedule, threading
from my_status import *
def superviorTask():
r = requests.get('https://medium.com')
if r.status_code == 200 and len(r.text) > 0:
status = STATUS_LIVE
else:
status = STATUS_INACTIVE # record to my database that the supervisor is running
def threadTask(job_function):
job_thread = threading.Thread(target=job_function)
job_thread.start()
# start schedule
schedule.every(10).minutes.do(threadTask, superviorTask)
while 1:
schedule.run_pending()
time.sleep(1)
and when I try below:
#!/usr/bin/env python3.6
from datetime import datetime
import time
import sys
import schedule
schedule.every(10).seconds.do(print("schedule tests!"))
while True:
print(f"You run test program using Supervisor at {datetime.now().strftime('%H-%M-%S')}.")
print("Good job Darek :)!!!")
print("...")
time.sleep(5)
I have error:
import schedule ModuleNotFoundError: No module named 'schedule'
Traceback (most recent call last): File
"/home/darek/PycharmProjects/Small_programs/shell_scripts/test/test_s.py",
line 5, in import schedule ModuleNotFoundError: No module
named 'schedule' Traceback (most recent call last):
File"/home/darek/PycharmProjects/Small_programs/shell_scripts/test/test_s.py",
line 5, in import schedule ModuleNotFoundError: No module
named 'schedule'
The default version of schedule package in python 2.7
If you have 3. version forward, you need to specify it:
pip3 install schedule (at CMD)
I created an executable using pyinstaller. My scripts all run fine but when i run the executable i get the error below. It appears it doesnt like the mysql connector in DBconnection.py. Any help is appreciated.
Traceback (most recent call last):
File "main.py", line 8, in
File "c:\programdata\anaconda3\lib\site-packages\PyInstaller\loader\pyimod03_i
mporters.py", line 631, in exec_module
File "DBconnection.py", line 2, in
ModuleNotFoundError: No module named 'mysql'
[4780] Failed to execute script main
DBconnection.py:
from configparser import ConfigParser
from mysql.connector import MySQLConnection
import Global
def create_db_connection(filename= 'my.ini', section= 'dbconnection'):
parser = ConfigParser()
parser.read(filename)
db = {}
if parser.has_section(section):
items = parser.items(section)
for item in items:
db[item[0]] = item[1]
else:
raise Exception('{0} not found in the {1} file'.format(section. filename))
#global conn
Global.conn = MySQLConnection(**db)
print(Global.conn)
def close_db_connection():
Global.conn.close()
main.py:
from servers import updateservers
from policies import updatepolicies
from updateguardpoints import updateguardpoints
from activities import updateactivities
from guardpointstatus import updateguardpointstatus
from application import updateapplication
from application_servers import updateapplication_servers
from DBconnection import create_db_connection
from DBconnection import close_db_connection
from TrouxConnection import TrouxConnection
from TrouxConnection import CloseTrouxConnection
import Global
create_db_connection()
TrouxConnection()
updateservers()
updatepolicies()
updateguardpoints()
updateactivities()
updateguardpointstatus()
updateapplication()
updateapplication_servers()
CloseTrouxConnection()
close_db_connection()
reinstalled mysql - connector using PIP. Issue resolved
I built an API endpoint using Flask, where data is collected and combined from other APIs. In order to do this efficiently, I use multiprocess. To keep control, I want to log all steps using Google Stackdriver.
For some reason, I keep getting errors when using Google Stackdriver within my multiprocess environment. The error and later warning I get within my MWE is the following:
Pickling client objects is explicitly not supported.
Clients have non-trivial state that is local and unpickleable.
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\...\Anaconda3\lib\multiprocessing\spawn.py", line 105, in spawn_main
exitcode = _main(fd)
File "C:\Users\...\Anaconda3\lib\multiprocessing\spawn.py", line 115, in _main
self = reduction.pickle.load(from_parent)
EOFError: Ran out of input
Minimal Working Example (excluded Flask/API for simplicity):
project_name = project_name = 'budget_service'
message = 'This is a test'
labels = {
'deployment': 'develop',
'severity': 'info'
}
# Import libs
from google.cloud import logging
import multiprocessing as mp
# Initialize logging
logging_client = logging.Client()
logger = logging_client.logger(project_name)
# Function to write log
def writeLog(logger):
logger.log_text(
text = message,
labels = labels
)
print('logger succeeded')
def testFunction():
print('test')
# Run without mp
writeLog(logger)
# Run with mp
print(__name__)
if __name__ == '__main__':
try:
print('mp started')
# Initialize
manager = mp.Manager()
return_dict = manager.dict()
jobs = []
# Set up workers
worker_log1 = mp.Process(name='testFunction', target=testFunction, args=[])
worker_log2 = mp.Process(name='writeLog', target=writeLog, args=[logger])
# Store in jobs
jobs.append(worker_log1)
jobs.append(worker_log2)
# Start workers
worker_log1.start()
worker_log2.start()
for job in jobs:
job.join()
print('mp succeeded')
except Exception as err:
print(err)
Why is it not possible to combine multiprocessing with Google Stackdriver? What should I adjust (what do I understand poorly) to make this work?
As of today (04.2019), stackdriver logging still does not support multiprocessing. The solution is either to:
Make sure your processes are started in spawn mode rather than fork (default on *nix) which prevents sharing anything
Avoid sharing logging objects explicitly by configuring them separately in each process
Using fork multiprocessing is generally a bad idea with google libraries, stackdriver is not the only one causing problems.
I am trying to generate an HTML test report by using Selenium webdriver library HTMLTestRunner. I am using Python 3.4 version and I have a version related error. Refer following snippet.
import unittest`enter code here`
import HTMLTestRunner
from selenium import webdriver
from selenium.webdriver.common.alert import Alert
from selenium.webdriver.common.keys import Keys
class GreenlamTest(unittest.TestCase):
#classmethod
def setUp(cls):
cls.driver=webdriver.Firefox()
cls.driver.implicitly_wait(30)
cls.driver.maximize_window()
cls.driver.get('https://www.google.co.in')
def test_checkTitle(self):
assert "Google" in self.driver.title
def test_searchtest(self):
driver = self.driver
elem = self.driver.find_element_by_name("q")
elem.send_keys("pycon")
elem.send_keys(Keys.RETURN)
#classmethod
def tearDown(cls):
cls.driver.quit()
if __name__ == '__main__':
HTMLTestRunner.main
Output
Finding files... done.
Traceback (most recent call last):
File "C:\Users\vaibhav\Desktop\Selenium Softwares\eclipse-jee-luna-SR2-win32-x86_64\eclipse\plugins\org.python.pydev_4.0.0.201504132356\pysrc\pydev_runfiles.py", line 468, in __get_module_from_str
mod = __import__(modname)
File "C:\Users\vaibhav\Desktop\Selenium Softwares\Practice\pythondemo\Htmlreport.py", line 2, in <module>
import HTMLTestRunner
File "C:\Users\vaibhav\Desktop\Selenium Softwares\Practice\pythondemo\HTMLTestRunner.py", line 94, in <module>
import StringIO
ImportError: No module named 'StringIO'
ERROR: Module: Htmlreport could not be imported (file: C:\Users\vaibhav\Desktop\Selenium Softwares\Practice\pythondemo\Htmlreport.py).
Importing test modules ... done.
----------------------------------------------------------------------
Ran 0 tests in 0.000s
OK
You are getting error because your folder name Selenium Softwares has a space in it where your Htmlreport is present. Replace the folder name to exclude space out of it, probably an underscore or camelCase, etc... like this Selenium_Softwares . Here's how -
file: C:\Users\vaibhav\Desktop\Selenium_Softwares\Practice\pythondemo\Htmlreport.py)
Hope this helps.