Hello jr data engineer here!
For some strange reason my task_fail_slack_alert module is triggering the Slack API request a ridiculous amount of times, which is then showing up in our Slack channel that many times and is really annoying. My module should only run and show up in in Slack channel the same amount as the number of tasks that failed.
What am I missing?
import os
from airflow.models
import Variable
import json import requests
def get_channel_name():
channel = '#airflow_alerts_local'
env = Variable.get('env', None)
if env == 'prod':
channel = '#airflow_alerts'
elif env == 'dev':
channel = '#airflow_alerts_dev'
return channel
def task_fail_slack_alert(context):
webhook_url = os.environ.get('SLACK_URL')
slack_data = {
'channel': get_channel_name(),
'text':
""" :red_circle: Task Failed.
*Task*: {task}
*Dag*: {dag}
*Execution Time*: {exec_date}
*Log Url*: {log_url}
""".format(
task=context.get('task_instance').task_id,
dag=context.get('task_instance').dag_id,
ti=context.get('task_instance'),
exec_date=context.get('execution_date'),
log_url=context.get('task_instance').log_url,
)}
response = requests.post(webhook_url, data=json.dumps(slack_data),
headers={'Content-Type': 'application/json'})
if response.status_code != 200:
raise ValueError( 'Request to slack returned an error %s,
the response is:\n%s'(response.status_code, response.text))
task_fail_slack_alert(context)
This is how I have it showing up in the arguments for each dag:
default_args = {
'on_failure_callback': task_fail_slack_alert,
}
The code you provided is recursive:
def task_fail_slack_alert(context):
......
task_fail_slack_alert(context)
Remove the recursion as it's not needed.
Related
The slackbot is not showing as active and not responding. I have used ngrok to set up tunneling from my localhost to allow for the slack bot to be verified. On slack it shows that the request URL has been verified. I have also subscribed to slack events.
I am following, https://medium.com/developer-student-clubs-tiet/how-to-build-your-first-slack-bot-in-2020-with-python-flask-using-the-slack-events-api-4b20ae7b4f86, to get this bot working. Any help would be great. Thanks.
from flask import Flask, Response
from slackeventsapi import SlackEventAdapter
import os
from threading import Thread
from slack import WebClient
# This `app` represents your existing Flask app
app = Flask(__name__)
greetings = ["hi", "hello", "hello there", "hey"]
SLACK_SIGNING_SECRET = os.environ['SLACK_SIGNING_SECRET']
slack_token = os.environ['SLACK_BOT_TOKEN']
VERIFICATION_TOKEN = os.environ['VERIFICATION_TOKEN']
#instantiating slack client
slack_client = WebClient(slack_token)
# An example of one of your Flask app's routes
#app.route("/")
def event_hook(request):
json_dict = json.loads(request.body.decode("utf-8"))
if json_dict["token"] != VERIFICATION_TOKEN:
return {"status": 403}
if "type" in json_dict:
if json_dict["type"] == "url_verification":
response_dict = {"challenge": json_dict["challenge"]}
return response_dict
return {"status": 500}
return
slack_events_adapter = SlackEventAdapter(
SLACK_SIGNING_SECRET, "/slack/events", app
)
#slack_events_adapter.on("app_mention")
def handle_message(event_data):
def send_reply(value):
event_data = value
message = event_data["event"]
if message.get("subtype") is None:
command = message.get("text")
channel_id = message["channel"]
if any(item in command.lower() for item in greetings):
message = (
"Hello <#%s>! :tada:"
% message["user"] # noqa
)
slack_client.chat_postMessage(channel=channel_id, text=message)
thread = Thread(target=send_reply, kwargs={"value": event_data})
thread.start()
return Response(status=200)
# Start the server on port 3000
if __name__ == "__main__":
app.run(port=3000)
This was the error code that I was running into:
urllib.error.URLError: <urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1091)>.
This brought me to this StackOverflow thread:
Scraping: SSL: CERTIFICATE_VERIFY_FAILED error for http://en.wikipedia.org
Running the certification file in the python folder allowed the slack bot to interact with the channel. The bot is now working but does not show as active.
I have a ChatBot application running, just want to hook this application with Slack-api as it's interface.
I used Slack RTM and maintained user-session with its slack user-id.
finally solved and written a client(API) which can easily connect to any conversation engine.
Github repo link-
https://github.com/csemanmohan/Slack_api_client
import time
import re
from slackclient import SlackClient
import requests
# 'url', chatbot endpoint and 'slack_token' is slack application user-access-token
url = "http://127.0.0.1:****/*******/v2/api"
slack_token = "xoxb-**********-***********-*************lipO8hoI"
# instantiate Slack client
slack_client = SlackClient(slack_token)
# starterbot's user ID in Slack: value is assigned after the bot starts up
starterbot_id = None
# constants
RTM_READ_DELAY = 1 # 1 second delay between reading from RTM
EXAMPLE_COMMAND = "do"
MENTION_REGEX = "^<#(|[WU].+?)>(.*)"
def parse_bot_commands(slack_events):
"""
Parses a list of events coming from the Slack RTM API to find bot commands.
If a bot command is found, this function returns a tuple of command and channel.
If its not found, then this function returns None, None.
"""
# below var msg and channel_var will be used/
# when no trigger(#app-name) passed from application
msg = ""
channel_def = ""
for event in slack_events:
if event["type"] == "message" and not "subtype" in event:
msg = event["text"]
channel_def = event["channel"]
user_id, message = parse_direct_mention(event["text"])
print("there is an event here...", user_id, message)
if user_id == starterbot_id:
return message, event["channel"]
channel_def = channel_def
return msg, channel_def
def parse_direct_mention(message_text):
"""
Finds a direct mention (a mention that is at the beginning) in message text
and returns the user ID which was mentioned. If there is no direct mention, returns None
"""
matches = re.search(MENTION_REGEX, message_text)
# the first group contains the username, the second group contains the remaining message
return (matches.group(1), matches.group(2).strip()) if matches else (None, None)
def handle_command(command, channel):
"""
Executes bot command if the command is known
"""
# Default response is help text for the user
default_response = "Not sure what you mean. Try *{}*.".format(EXAMPLE_COMMAND)
# Implemented below code-snippet for making API call to ChatBot
input_text = command
payload = {"text": input_text, "email": "manmohan#m******.com"}
headers = {'content-type': "application/json"}
resp = requests.request("POST", url, json=payload, headers=headers)
result = eval(resp.json())
print("result is: ", result)
response = result['text']
# Sends the response back to the channel
slack_client.api_call(
"chat.postMessage",
channel=channel,
text=response or default_response
)
if __name__ == "__main__":
if slack_client.rtm_connect(with_team_state=False):
print("Starter Bot connected and running!")
# Read bot's user ID by calling Web API method `auth.test`
starterbot_id = slack_client.api_call("auth.test")["user_id"]
while True:
command, channel = parse_bot_commands(slack_client.rtm_read())
if command:
handle_command(command, channel)
time.sleep(RTM_READ_DELAY)
else:
print("Connection failed. Exception traceback printed above.")
I am trying to learn some Http Server in an udacity online academy. The thing is that the folllowing code is triggering the error Message: Unsupported method ('POST'). Error 501 Python:
from http.server import HTTPServer, BaseHTTPRequestHandler
from urllib.parse import parse_qs
class MessageHandler(BaseHTTPRequestHandler):
def do_POST(self):
# 1. How long was the message?
length = int(self.headers.get('Content-length', 0))
# 2. Read the correct amount of data from the request.
data = self.rfile.read(length).decode()
# 3. Extract the "message" field from the request data.
message = parse_qs(data)["message"][0]
# Send the "message" field back as the response.
self.send_response(200)
self.send_header('Content-type', 'text/plain; charset=utf-8')
self.end_headers()
self.wfile.write(message.encode())
if __name__ == '__main__':
server_address = ('', 8000)
httpd = HTTPServer(server_address, MessageHandler)
httpd.serve_forever()
Which Python? Your code is correct. Tested it right now, it sends the response.
The only modification I've made is
#message = parse_qs(data)["message"][0]
message = 'hello'
Client code:
import requests
res = requests.post('http://localhost:8000/abc', data = {'key':'value'})
print(res)
Client gets 200 response
I am fetch all child account from the Master AWS Account by boto3 Organization.
Code is working fine. I am able to get child account list.
But if you run my AWS Lambda function again then it fail to get Child Accounts.
Getting following error:
Error while getting AWS Accounts : An error occurred (TooManyRequestsException) when calling the ListAccounts operation: AWS Organizations can't complete your request because another request is already in progress. Try again later
After 20 to 30 minutes, I can see my code work for once and again raise above exception.
I am Run this code by AWS Gateway + AWS Lambda.
Any idea?
Code:
import boto3
class Organizations(object):
"""AWS Organization"""
def __init__(self, access_key, secret_access_key, session_token=None):
self.client = boto3.client('organizations',
aws_access_key_id=access_key,
aws_secret_access_key=secret_access_key,
aws_session_token=session_token
)
def get_accounts(self, next_token=None, max_results=None):
"""Get Accounts List"""
if next_token and max_results:
result = self.client.list_accounts(NextToken=next_token,
MaxResults=max_results)
elif next_token:
result = self.client.list_accounts(NextToken=next_token)
elif max_results:
result = self.client.list_accounts(MaxResults=max_results)
else:
result = self.client.list_accounts()
return result
class AWSAccounts(object):
""" Return AWS Accounts information. """
def get_aws_accounts(self, access_key, secret_access_key, session_token):
""" Return List of AWS account Details."""
org_obj = Organizations(access_key=access_key,
secret_access_key=secret_access_key,
session_token=session_token)
aws_accounts = []
next_token = None
next_result = None
while True:
response = org_obj.get_accounts(next_token, next_result)
for account in response['Accounts']:
account_details = {"name": account["Name"],
"id": account["Id"],
"admin_role_name": self.account_role_name
}
aws_accounts.append(account_details)
if "NextToken" not in response:
break
next_token = response["NextToken"]
return aws_accounts
By Exception Handling, my code is running successfully.
Catch TooManyRequestsException exception by ClientError exception and retry to call AWS list_accounts API by boto3.
We can add time sleep of 0.1 seconds.
Code:
class AWSAccounts(object):
""" Return AWS Accounts information. """
def get_accounts(self, next_token=None, max_results=None):
"""Get Accounts List"""
# If Master AWS account contain more child accounts(150+) then
# Too-Many-Request Exception is raised by the AWS API(boto3).
# So to fix this issue, we are calling API again by Exception Handling.
result = None
while True:
try:
if next_token and max_results:
result = self.client.list_accounts(NextToken=next_token,
MaxResults=max_results)
elif next_token:
result = self.client.list_accounts(NextToken=next_token)
elif max_results:
result = self.client.list_accounts(MaxResults=max_results)
else:
result = self.client.list_accounts()
except botocore.exceptions.ClientError as err:
response = err.response
print("Failed to list accounts:", response)
if (response and response.get("Error", {}).get("Code") ==
"TooManyRequestsException"):
print("Continue for TooManyRequestsException exception.")
continue
break
return result
Configure your boto3 client to use the built-in standard retry mode:
import boto3
from botocore.config import Config
config = Config(
retries = {
'max_attempts': 10,
'mode': 'standard'
}
)
ec2 = boto3.client('ec2', config=config)
Per the documentation, the default mode is 'legacy' which doesn't handle TooManyRequestsException.
See boto3 documentation about retry configuration here: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html
The Python3 fetch_token method in this library does not check the response status before consuming the response. If the API call it makes fails, then the response will be invalid and the script crashes. Is there something I can set so that an exception will be raised on a non-success response before the library can read the response?
import requests
from requests.auth import HTTPBasicAuth
from requests_oauthlib import OAuth2Session
from oauthlib.oauth2 import BackendApplicationClient
from oauthlib.oauth2 import OAuth2Error
AUTH_TOKEN_URL = "https://httpstat.us/500" # For testing
AUTH = HTTPBasicAuth("anID", "aSecret")
CLIENT = BackendApplicationClient(client_id="anID")
SCOPES = "retailer.orders.write"
MAX_API_RETRIES = 4
class MyApp:
def __init__(self):
"""Initialize ... and obtain initial auth token for request"""
self.client = OAuth2Session(client=CLIENT)
self.client.headers.update(
{
"Content-Type": "application/json"
}
)
self.__authenticate()
def __authenticate(self):
"""Obtain auth token."""
server_errors = 0
# This needs more work. fetch_token is not raising errors but failing
# instead.
while True:
try:
self.token = self.client.fetch_token(
token_url=AUTH_TOKEN_URL, auth=AUTH, scope=SCOPES
)
break
except (OAuth2Error, requests.exceptions.RequestException) as e:
server_errors = MyApp.__process_retry(
server_errors, e, None, MAX_API_RETRIES
)
#staticmethod
def __process_retry(errors, exception, resp, max_retries):
# Log and process retries
# ...
return errors + 1
MyApp() # Try it out
You can add a "compliance hook" that will be passed the Response object from requests before the library attempts to parse it, like so:
def raise_on_error(response):
response.raise_for_status()
return response
self.client.register_compliance_hook('access_token_response', raise_on_error)
Depending on exactly when you may get errors, you might want to do this with 'refresh_token_response' and/or 'protected_request' as well. See the docstring for the register_compliance_hook method for more info.