When work with Celery 4.3.0 Chains,
Code like this:
class BaseTask(Task):
def on_success(self, retval, task_id, args, kwargs):
pass
#app.task(bind=True, base=BaseTask)
def add(self, x, y):
return x+y
#app.task(bind=True, base=baseTask)
def add_1(self,x, y):
return x+y
task = group(add.s(1,3), add_1.s(2,4))()
group_id = task.id
I can get group_id by taks.children[0]._cache.get("group"),
How can I get group_id in BaseTask on_success functions?
You can find the id of the task’s group inside the app.task.Request object, made available through the Task object, like so:
class BaseTask(Task):
def on_success(self, retval, task_id, args, kwargs):
group_id = self.request.group
# rest of your code
Related
I found a workaround for my issue but I need to know why the first above case doesn't work.
I need to pass a parameter (reman_pk) to my view but when I try :
class RepairCreateView(LoginRequiredMixin, CreateView):
#property
def reman_pk(self):
return int(self.kwargs['reman_pk'])
[...]
success_url = reverse_lazy(
'reman:update-reman', kwargs={'pk': reman_pk})
[...]
... I got an error
django.urls.exceptions.NoReverseMatch: Reverse for 'update-reman' with keyword arguments '{'pk': <property object at 0x10c20bbd0>}' not found. 1 pattern(s) tried: ['reman/update/(?P[0-9]+)/$']
But when in the same class based view I use :
def get_success_url(self, **kwargs):
if kwargs != None:
return reverse_lazy('reman:update-reman', kwargs={'pk': self.reman_pk})
... it's OK : an int is well passed in my URL.
I tried to pass int(reman_pk) in the first method ... not better.
I've already use #property in the past and always got a value (int/str) and not property object.
EDIT (FULL views.py)
success_url = reverse_lazy...is commented. I must use def get_success_url( ... instead. Otherwise I get the above mentioned error.
class RepairCreateView(LoginRequiredMixin, CreateView):
#property
def reman_pk(self):
return int(self.kwargs['reman_pk'])
# success_url = reverse_lazy(
# 'reman:repairs-list', kwargs={'pk': reman_pk})
success_message = "Nouvelle réparation créée"
form_class = RepairCreateForm
template_name = 'reman/repair_create_form.html'
def get_context_data(self, *args, **kwargs):
context = super(RepairCreateView, self).get_context_data(
*args, **kwargs)
context['title'] = 'Nouveau Repair'
context['pk'] = self.reman_pk
return context
def get_initial(self):
reman = Reman.objects.get(pk=self.reman_pk)
return {'reman': reman}
def get_success_url(self, **kwargs):
return reverse_lazy('reman:repairs-list', kwargs={'pk': self.reman_pk})
This is not related to django, this is related to python in general. When you want to access a class property within the class you always have to call self before!
class Tree:
fruits = 5
#property
def leafes(self):
return self.fruits * 5
def show_tree(self):
print(self.fruits)
print(self.leafes)
print(leafes) # THIS LINE WOULD ERROR
Edit after comment of OP
I don't know how to phrase this properly. Anyhow this keeps being a problem related to python and not to django. The reason is how classes work.
You probably know the def __init__(self): function. That is called when the class gets instanciated. After that function got called your class can use all the self attributes (class attributes). But class attributes like my fruits = 5 get assigned even before that def __init__(self) method is called. So all your assignments directly inside the body of the class do not have self yet.
class Tree:
fruits = 5
def __init__(self):
self.twigs = 10
self.weight = self.twigs + self.fruits # THIS WORKS
class Tree:
fruits = 5
weight = self.twigs + fruits # THIS DOES NOT WORK
def __init__(self):
self.twigs = 10
Last example does not work because at the moment you want to assign weight = self.twigs + fruits your class's __init__ function was not called yet. So you can not use self at that place.
I have a collection of ever more specialized classes which correspond to collections of the same kind of data (temperature, density, etc) but for different drifts, for example, one subclass has dimensions (nx, ny) and a different suclass has dimensions (ncv), and I want to reflect that in the docstrings, for having a better documentation using Sphinx.
After reading many very useful threads here in Stack Overflow, I have arrived to this model:
import numpy as np
from functools import wraps
def class_decorator(cls):
import ipdb; ipdb.set_trace()
clsdict = {}
mro = cls.mro()
mro.reverse()
for tmp in mro[1:]: ##Ignore object class parent.
clsdict.update(tmp.__dict__)
for name, method in clsdict.items():
if hasattr(method, '__og_doc__'):
try:
method.__doc__ = method.__og_doc__.format(**clsdict)
except:
pass
else:
try:
method.__og_doc__ = method.__doc__
method.__doc__ = method.__doc__.format(**clsdict)
except:
pass
return cls
def mark_documentation(fn):
if not hasattr(fn, '__og_doc__'):
try:
fn.__og_doc__ = fn.__doc__
except:
pass
#wraps(fn)
def wrapped(*args, **kwargs):
return fn(*args, **kwargs)
return wrapped
def documented_property(fn):
if not hasattr(fn, '__og_doc__'):
try:
fn.__og_doc__ = fn.__doc__
except:
pass
#wraps(fn)
def wrapped(*args, **kwargs):
return fn(*args, **kwargs)
prp= property(wrapped)
prp.__og_doc__ = fn.__og_doc__
return prp
#class_decorator
class Base(object):
_GRID_DIM = 'nx, ny'
_TYPE = 'BaseData'
def __init__(self, name):
self.name = name
def shape(self):
""" This docstring contains the type '{_TYPE}' of class."""
print('Simple')
def operation(self, a, b, oper=np.sum, **kwargs):
""" Test for functions with args and kwargs in {_TYPE}"""
return oper([a,b])
#classmethod
def help(cls, var):
try:
print(get(cls, var).__doc__)
except:
print("No docstring yet.")
#class_decorator
class Advanced(Base):
_GRID_DIM = 'ncv'
_TYPE = 'AdvancedData'
def __init__(self,name):
super().__init__(name)
#property
#mark_documentation
# #documented_property
def arkansas(self):
"""({_GRID_DIM}, ns): Size of Arkansaw."""
return 'Yeah'
I am aiming to get the correctly formatted docstring when I call the help method or I use Sphinx, so that:
> adv = Advanced('ADV')
> adv.help("arkansas")
(ncv, ns): Size of Arkansaw.
> adv.help("operation")
Test for functions with args and kwargs in AdvancedData
I have managed to make it work so far, except for properties, because I assigned __og_doc__ to the function, but the property does not have that attribute. My last attempt at monkeypatching this, documented_property, fails because property is inmutable (as expected), and I cannot come up with any way to avoid this roadblock.
Is there any way around this problem?
All I am trying to do is to create a reusable ambda component where I can pass parameters to the class so that the Lambda can do different things, based on input param.
I am using CDK in python to deploy the stack. I would like to create a parallel stepfunction, where I can pass the same lambda using different param/payload so they can be branches.
I am running the following code:
from aws_cdk import (
# Duration,
Stack,
# aws_sqs as sqs,
aws_stepfunctions as _stepfunctions,
aws_stepfunctions_tasks as _stepfunctions_tasks,
aws_lambda as _lambda,
)
from constructs import Construct
class LambdaJob(_stepfunctions.StateMachineFragment):
def __init__(self, parent, id, *, jobTypeParam):
super().__init__(parent, id)
existingFunc = _lambda.Function.from_function_arn(self, "ExistingLambdaFunc", function_arn="arn:aws:lambda:us-east-1:95842$$$$$:function:dummyFunction")
lambda_invoked = _stepfunctions_tasks.LambdaInvoke(self, "someID", lambda_function=existingFunc)
wait_10_seconds = _stepfunctions.Wait(self, "Wait for 10 seconds",
time=_stepfunctions.WaitTime.duration(Duration.seconds(10))
)
self.start_state = wait_10_seconds
class StepfunctionsClasStack(Stack):
def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
super().__init__(scope, construct_id, **kwargs)
test_lambda_1 = LambdaJob(self, "Quick")
#state_machine = _stepfunctions.StateMachine(self, "TestStateMachine",
# definition=LambdaJob(self, "Quick"),
# # role=marketo_role
# )
However i keep getting the following error:
TypeError: Can't instantiate abstract class LambdaJob with abstract methods end_states, start_state
Any thoughts on what I am doing wrong ?
Thanks
Solved using the override:
self._start_state = parallel_state#wait_10_seconds
self._end_states = [chain]#[lambda_invoked]
def start_state(self):
return self._start_state
def end_states(self):
return self._end_states
I want a callback to get called whenever a certain attribute of object A is changed.
I'm aware that this question is related to Observer Pattern and descriptors in Python. However, it seems descriptors could only detect explicit changes via dot access.
For instance:
class Observer(object):
def __init__(self, callback=None):
self.__callback = callback
def __set_name__(self, owner, name):
self.__name = name
def __set__(self, obj, value):
obj.__dict__[self.__name] = value
self.trigger()
def __get__(self, obj, type=None):
return obj.__dict__.get(self.__name)
def trigger(self):
self.__callback()
def hello():
print('hello')
class MyClass:
data = Observer(hello)
a = MyClass()
a.data = [[1],2,3]
a.data.append(4)
a.data[0][0] = -1
In the above code, the callback is only called once for the initialization of the data. However, I want it to be called 3 times. I'm not tied to using descriptors but I do want the method to work on any data types, such as list, dict and etc.
I would like to set some target paths as global variables in Luigi.
The reason is that the target paths I'm using are based on the last run of a given numerical weather prediction (NWP), and it takes some time to get the value. Once I have checked which is the last run I create a path in which I will put several target files (with the same parent folder).
I'm currently repeating a similar call to get the value of the parent path for several tasks, and it would be much efficient to set this path as a global variable. I have tried to define global variable from within one function (get_target_path) called by a luigi class, but it looks like the global variable doesn't persist when I get back to Luigi pipeline.
This is moreover how my code looks like:
class GetNWP(luigi.Task):
"""
Download the NWP data.
"""
product_id = luigi.Parameter()
date = luigi.Parameter(default=datetime.today().strftime('%Y%m%d'))
run_hr = luigi.Parameter(default='latest')
def requires(self):
return None
def output(self):
path = get_target_path(self.product_id, self.date, self.run_hr,
type='getNWP')
return luigi.LocalTarget(path)
def run(self):
download_nwp_data(self.product_id, self.date, self.run_hr)
class GetNWP_GFS(luigi.Task):
"""
GFS data.
"""
product_id = luigi.Parameter()
date = luigi.Parameter(default=datetime.today().strftime('%Y%m%d'))
run_hr = luigi.Parameter(default='latest')
def requires(self):
return None
def output(self):
path = get_target_path(self.product_id_PV, self.date, self.run_hr,
type='getNWP_GFS')
return luigi.LocalTarget(path)
def run(self):
download_nwp_data(self.product_id, self.date, self.run_hr,
type='getNWP_GFS')
class Predict(luigi.Task):
"""
Create forecast.
"""
product_id = luigi.Parameter(default=None)
date = luigi.Parameter(default=datetime.today().strftime('%Y%m%d'))
run_hr = luigi.Parameter(default='latest')
horizon = luigi.Parameter(default='DA')
def requires(self):
return [
GetNWP_GFS(self.product_id, self.date, self.run_hr),
GetNWP(self.product_id, self.date, self.run_hr)
]
def output(self):
path = get_target_path(self.product_id, self.date, self.run_hr,
type='predict', horizon=self.horizon)
return luigi.LocalTarget(path)
def run(self):
get_forecast(self.product_id, self.date, self.run_hr)
The function get_target_path defines a target path based on the input parameters. I would like this function to set global variables that would be accessible from Luigi. For example as follows (just the code for the getNWP task):
def get_target_path(product_id, date, run_hr, type=None, horizon='DA'):
"""
Obtain target path.
"""
if type == 'getNWP_GFS':
if 'path_nwp_gfs' in globals():
return path_nwp_gfs
else:
...
elif type == 'getNWP':
if 'path_nwp_model' in globals():
return path_nwp_model
else:
filename = f'{nwp_model}_{date}_{run_hr}_{horizon}.{ext}'
path = Path(db_dflt['app_data']['nwp_folder'])
create_directory(path)
global path_nwp_model
path_nwp_model = Path(path) / filename
elif type == 'predict':
if 'path_predict' in globals():
return path_predict
else:
...
The global variable defined in this function doesn't exist when I'm back to Luigi.
Any ideas on how to solve this problem will be appreciated!
As it seems there is no built in method to store the paths of Luigi's targets I finally decided to create a class which holds all the information related to Luigi's targets/paths. This class is used within Luigi's Tasks when calling external functions which need to know which are the target paths.
This class is imported in the main luigy script, and instantiated before defining the Tasks:
from .utils import Targets
paths = Targets()
class GetNWP(luigi.Task):
"""Download NWP data required to prepare the prediction."""
product_id = luigi.Parameter()
date = luigi.Parameter(default=datetime.today().strftime('%Y%m%d'))
run_hr = luigi.Parameter(default='latest')
def requires(self):
return GetProductInfo(self.product_id)
def output(self):
path = paths.getpath_nwp(self.product_id, self.date, self.run_hr)
path_gfs = paths.getpath_nwp_GFS(self.product_id, self.date, self.run_hr)
return [luigi.LocalTarget(path),
luigi.LocalTarget(path_gfs)]
def run(self):
download_nwp_data(self.product_id, date=self.date, run_hr=self.run_hr,
paths=paths, nwp_model=paths.nwp_model)
download_nwp_data(self.product_id, date=self.date, run_hr=self.run_hr,
paths=paths, nwp_model=paths.gfs_model)
class Predict(luigi.Task):
"""Create forecast based on the product information and NWP data."""
product_id = luigi.Parameter()
date = luigi.Parameter(default=datetime.today().strftime('%Y%m%d'))
run_hr = luigi.Parameter(default='latest')
def requires(self):
return GetNWP(self.product_id, self.date, self.run_hr)
def output(self):
path = paths.getpath_predict(self.product_id, self.date, self.run_hr)
path_gfs = paths.getpath_predict_GFS(self.product_id, self.date,
self.run_hr)
return [luigi.LocalTarget(path),
luigi.LocalTarget(path_gfs)]
def run(self):
get_forecast(product_id=self.product_id, date=self.date,
run_hr=self.run_hr, paths=paths, nwp_model=paths.nwp_model)
get_forecast(product_id=self.product_id, date=self.date,
run_hr=self.run_hr, paths=paths, nwp_model=paths.gfs_model)
where Targets class has the following structure:
class Targets:
"""Store Luigi's target paths."""
def __init__(self):
"""Initialize paths and variables."""
self.path1 = None
self.path2 = None
self.path3 = None
def update_object(self, product_id, date=None, run_hr=None):
"""Update object based on inputs."""
if self.prod_id is None:
self.prod_id = product_id
if self.path_1 is None:
self.get_path_1(product_id)
if self.path_2 is None:
self.get_path_2(product_id)
if self.path_3 is None:
self.get_path_3(product_id)
def get_path_1(self, product_id, ...)
"""Generate a path 1 for a luigi Task."""
... define self.path_1...
def get_path_2(self, product_id, ...)
"""Generate a path 2 for a luigi Task."""
... define self.path_2...
def get_path_3(self, product_id, ...)
"""Generate a path 3 for a luigi Task."""
... define self.path_3...
The main idea is to set the target paths only one time and use them from within each Luigi task as input parameters. This allows to:
Perform task more rapidly, and
Avoid errors if a target path changes due to new NWP oavailable.
You can use mixins if you'd like, but keep in mind that luigi tasks can inherit instance methods and parameters.
import os
import luigi
LUIGI_BASE_PATH='/path/to/luigi/dir'
class BaseTask(luigi.Task)
product_id = luigi.Parameter()
date = luigi.Parameter(default=datetime.today().strftime('%Y%m%d'))
run_hr = luigi.Parameter(default='latest')
def get_path_dynamic(self):
return os.path.join(LUIGI_BASE_PATH,
self.__class__.__name__,
self.product_id,
...)
def output(self):
return luigi.LocalTarget(self.get_path_dynamic())
class Predict(BaseTask):
def run(self):
...
The added benefit is that you don't need to redefine the same parameters, and the
the child task's name (Predict or GetNWP) will be inserted into the output path. I'm not sure how the path1, path2 etc attributes relate to the gpath_nwp() and similar functions since their definitions aren't included in the example, but you can mimic the same functionality using the #property decorator for defining getters and setters.