I am working on an API which will process videos and can take long time to complete. Earlier I build this functionality with qcluster and now I want to use celery for same purpose.
Here is the example of what I did with qcluster API-
class MyClass(APIView):
def post(self, request):
task_id = request.data['task_id']
if task_id: # will check if task id is present or not to get result for that task id
# concatenation process in queue
task_result = result(task_id)
if type(task_result) == dict:
return JsonResponse(task_result)
elif task_result == None:
return JsonResponse({
'message': "video is in process !",
'data': None,
'task_id': task_id,
'status': True
})
task_id = async_task( #task id will be generated here
# concatenate videos with translation
'video.editor.speed_vid.speed_up_vid_qc', #function path and payload
input_path=request.data["video_url"],
speed_factor = request.data["speed_factor"],
start = request.data["start"],
end = request.data["end"],
)
return JsonResponse({
'message': "speed_vid started ! "
"please use task_id to get result of the video ",
'data': None,
'task_id': task_id,
'status': True
})
So, With this 1 API I was able to create task and return the result without worrying about the time
Now I want to do the same with Celery is it possible ? if yes then any material or example ?
Related
Hey guys so I have this function in python that is async and after finishing returns a value but this function is added as a task as many times as there is items in a certain list the thing is I dont seem to be able to get the values returned from such functions after asyncio.wait() finishes running.
The code (I removed most of the bloat so this is a simplified code):
async def transcribe(s3_path: str, file_name: str):
for attempt in range(MAX_WAIT_ATTEMPTS):
# Do some stuff
await asyncio.sleep(WAIT_TIME)
return "value 1", "value 2"
async def transcribe_all_videos(videos_list: List[str], user_id: str):
tasks = []
for video in videos_list:
tasks.append(
asyncio.get_event_loop().create_task(transcribe("some path", "some video"))
)
result = await asyncio.wait(tasks)
# Extract result return data
return result
What I tried:
I tried running result[0].result()
result.result()
None of those seem to work and the error is always in the lines of 'tuple' object has no attribute 'result'
asyncio.wait returns done and pending tasks. So iterate over done tasks and use the .result() method:
import asyncio
async def transcribe(s3_path: str, file_name: str):
await asyncio.sleep(1)
return "value 1", "value 2"
async def transcribe_all_videos(videos_list, user_id):
tasks = []
for video in videos_list:
tasks.append(
asyncio.get_event_loop().create_task(
transcribe("some path", "some video")
)
)
done, pending = await asyncio.wait(tasks)
for r in done:
print(r.result())
asyncio.run(transcribe_all_videos(["xxx"], 1))
Prints:
('value 1', 'value 2')
I have a celery periodic task implemented to send email to users on certain date set on the model,How can i write the Pytest and mock for the following celery task with all the necessary argumuments.sending_email_function is function used in my task which accepts the arguments mentioned in the code.
#app.task(bind=True)
def send_email(self):
send_date = timezone.now()
records = Model.object.filter(send_date=send_date)
for record in records:
template_data = {
'homepage_url': settings.WWW_ROOT,
'domain_name': settings.DOMAIN_NAME,
'inq': record.inq,
'name': record.name,
'created': record.created,
'date': record.send_date,
}
user_emails = []
if record.send_email_to_users: **Boolean Flag on model**
user_emails = []
for user in record.users.all():
user_emails.append(user.email)
sending_email_function(
subject_template_path='',
body_template_path='',
template_data=template_data,
to_email_list=user_emails,
fail_silently=False,
content_subtype='html'
)
I would like to run the tasks and return the result immediately
tasks = [
asyncio.create_task(self.start_update(update, spec))
for update, spec in zip(updates, specs)
]
return Report(updates=updates,job_task=asyncio.gather(*tasks))
Now I want to change the logic and send a slack msg after my tasks will be completed, so I changed the code:
tasks = [
asyncio.create_task(self._start_update(update, spec))
for update, spec in zip(updates, specs)
]
task = asyncio.create_task(self.notify_as_completed(tasks))
return Report(updates=updates,job_task=asyncio.gather(task))
--------------------------------
async def notify_as_completed(self, tasks) -> None:
try:
execute_response = await asyncio.gather(*tasks)
await self._slack_service.send_msg(execute_response)
except Exception as ex:
logging.error(f"wasn't able to send a slack msg: {ex}")
Is it a good pattern or there is a better way?
Hello jr data engineer here!
For some strange reason my task_fail_slack_alert module is triggering the Slack API request a ridiculous amount of times, which is then showing up in our Slack channel that many times and is really annoying. My module should only run and show up in in Slack channel the same amount as the number of tasks that failed.
What am I missing?
import os
from airflow.models
import Variable
import json import requests
def get_channel_name():
channel = '#airflow_alerts_local'
env = Variable.get('env', None)
if env == 'prod':
channel = '#airflow_alerts'
elif env == 'dev':
channel = '#airflow_alerts_dev'
return channel
def task_fail_slack_alert(context):
webhook_url = os.environ.get('SLACK_URL')
slack_data = {
'channel': get_channel_name(),
'text':
""" :red_circle: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
)}
response = requests.post(webhook_url, data=json.dumps(slack_data),
headers={'Content-Type': 'application/json'})
if response.status_code != 200:
raise ValueError( 'Request to slack returned an error %s,
the response is:\n%s'(response.status_code, response.text))
task_fail_slack_alert(context)
This is how I have it showing up in the arguments for each dag:
default_args = {
'on_failure_callback': task_fail_slack_alert,
}
The code you provided is recursive:
def task_fail_slack_alert(context):
......
task_fail_slack_alert(context)
Remove the recursion as it's not needed.
I have a ChatBot application running, just want to hook this application with Slack-api as it's interface.
I used Slack RTM and maintained user-session with its slack user-id.
finally solved and written a client(API) which can easily connect to any conversation engine.
Github repo link-
https://github.com/csemanmohan/Slack_api_client
import time
import re
from slackclient import SlackClient
import requests
# 'url', chatbot endpoint and 'slack_token' is slack application user-access-token
url = "http://127.0.0.1:****/*******/v2/api"
slack_token = "xoxb-**********-***********-*************lipO8hoI"
# instantiate Slack client
slack_client = SlackClient(slack_token)
# starterbot's user ID in Slack: value is assigned after the bot starts up
starterbot_id = None
# constants
RTM_READ_DELAY = 1 # 1 second delay between reading from RTM
EXAMPLE_COMMAND = "do"
MENTION_REGEX = "^<#(|[WU].+?)>(.*)"
def parse_bot_commands(slack_events):
"""
Parses a list of events coming from the Slack RTM API to find bot commands.
If a bot command is found, this function returns a tuple of command and channel.
If its not found, then this function returns None, None.
"""
# below var msg and channel_var will be used/
# when no trigger(#app-name) passed from application
msg = ""
channel_def = ""
for event in slack_events:
if event["type"] == "message" and not "subtype" in event:
msg = event["text"]
channel_def = event["channel"]
user_id, message = parse_direct_mention(event["text"])
print("there is an event here...", user_id, message)
if user_id == starterbot_id:
return message, event["channel"]
channel_def = channel_def
return msg, channel_def
def parse_direct_mention(message_text):
"""
Finds a direct mention (a mention that is at the beginning) in message text
and returns the user ID which was mentioned. If there is no direct mention, returns None
"""
matches = re.search(MENTION_REGEX, message_text)
# the first group contains the username, the second group contains the remaining message
return (matches.group(1), matches.group(2).strip()) if matches else (None, None)
def handle_command(command, channel):
"""
Executes bot command if the command is known
"""
# Default response is help text for the user
default_response = "Not sure what you mean. Try *{}*.".format(EXAMPLE_COMMAND)
# Implemented below code-snippet for making API call to ChatBot
input_text = command
payload = {"text": input_text, "email": "manmohan#m******.com"}
headers = {'content-type': "application/json"}
resp = requests.request("POST", url, json=payload, headers=headers)
result = eval(resp.json())
print("result is: ", result)
response = result['text']
# Sends the response back to the channel
slack_client.api_call(
"chat.postMessage",
channel=channel,
text=response or default_response
)
if __name__ == "__main__":
if slack_client.rtm_connect(with_team_state=False):
print("Starter Bot connected and running!")
# Read bot's user ID by calling Web API method `auth.test`
starterbot_id = slack_client.api_call("auth.test")["user_id"]
while True:
command, channel = parse_bot_commands(slack_client.rtm_read())
if command:
handle_command(command, channel)
time.sleep(RTM_READ_DELAY)
else:
print("Connection failed. Exception traceback printed above.")