I have a problem with my AWS python code.
I am trying to send a post request to my code on AWS but I have key problems
My code on AWS
import json
import random
def lambda_handler(event, context):
name = surname = birthDate = favoriteFilm = password = ""
indexList = keysArray = []
setParams(event, name, surname, birthDate, favoriteFilm)
fillArray(keysArray, name, surname, birthDate, favoriteFilm)
arrayLength = len(keysArray)
while len(password)<6:
index = getRandomRangeIndex(arrayLength)
if index in indexList:
continue
password = password + keysArray[index]
indexList.append(index)
return {
'statusCode': 200,
'body': json.dumps(password)
}
def setParams(event, name, surname, birthDate, favoriteFilm):
name = event['first_name']
surname = event['last_name']
birthDate = event['d_o_b']
favoriteFilm = event['favorite_film']
def fillArray(keysArray, name, surname, birthDate, favoriteFilm):
for names in name.split():
keysArray.append(names)
keysArray.append(surname)
for dates in birthDate.split('-'):
keysArray.append(dates)
for films in favoriteFilm.split():
keysArray.append(films)
def getRandomRangeIndex(arrayLength):
return random.randint(0, arrayLength-1)
My Postman request header
{
"first_name": "John",
"last_name": "Smith",
"d_o_b": "1985-12-04",
"favorite_film": "Back to the Future"
}
My problem log
[ERROR] KeyError: 'first_name'
Traceback (most recent call last):
File "/var/task/password.py", line 7, in lambda_handler
setParams(event, name, surname, birthDate, favoriteFilm)
File "/var/task/password.py", line 24, in setParams
name = event['first_name']
I am not able to find any solution. How can I fix this problem? Thank you.
When you submit your json though api gateway, the event object that your function receives is different than what you submit. The format is shown in aws docs.
In your case the event will be something like this:
{
"resource": "/test",
"path": "/test",
"httpMethod": "POST",
#
# a banch of data removed for length
#
"body": "{\n \"first_name\": \"John\",\n \"last_name\": \"Smith\",\n \"d_o_b\": \"1985-12-04\",\n \"favorite_film\": \"Back to the Future\"\n}",
"isBase64Encoded": false
}
Thus, to get your actual data and you have to parse body you can use python's ast. You body is not json string, thus need to use ast:
import json
import random
import ast
def lambda_handler(event, context):
# overwrite event to keep using event in later parts of your code
event = ast.literal_eval(event['body'])
#....
Related
I'm trying to implement the redis caching in my API for the first time it is a simple fastapi application which is using openweather api to query some weather information and my intention is to cache each json response in the redis server. I'hve made it to work with single key city but with this approach validating query parameters not working with the error handling I put in the place.
caching.py
import sys
from datetime import timedelta
import json
import redis
from services.openweather_service import get_report
def redis_connect() -> redis.client.Redis:
try:
client = redis.Redis(
host="localhost",
port=6379,
db=0,
)
ping = client.ping()
if ping is True:
return client
except redis.ConnectionError:
print("Connection Error!")
sys.exit(1)
client = redis_connect()
def get_routes_from_cache(key: str) -> str:
"""Data from redis."""
val = client.get(key)
return val
def set_routes_to_cache(key: str, value: str) -> bool:
"""Data to redis."""
state = client.setex(
key,
timedelta(hours=24),
value=value,
)
return state
async def route_optima(city: str, state: str, country: str, units=None) -> dict:
location = {"city": city, "state": state, "country": country, "units": units}
# First it looks for the data in redis cache
data = get_routes_from_cache(key=json.dumps(location))
# print(data)
# print(type(data))
# If cache is found then serves the data from cache
if data is not None:
data = data.decode("UTF-8")
data_dict = json.loads(data)
print(data_dict)
print(type(data_dict))
data["cache"] = True
return data
else:
# If cache is not found then sends request to the OpenWeather API
data = await get_report(city, state, country, units)
# This block sets saves the respose to redis and serves it directly
data["cache"] = False
data = json.dumps(data)
state = set_routes_to_cache(key=json.dumps(location), value=json.dumps(data))
if state is True:
return json.dumps(data)
return data
Then I took a different approach by making the query params location = {"city": city, "state": state, "country": country, "units": units} as a key and the json response as a value but when app try to get the response from the cache here it got weird like right after passing the query params dict into the json.dumps it gives me the bytes object and so I decode it with decode("utf-8") but instead of coming back as a dict it gives me <class 'str'> type object. There I am lost... can anyone help me out here?
weather_api.py
from typing import Optional
import fastapi
from fastapi import Depends
from models.location import Location
from infrastructure.caching import route_optima
from models.validation_error import ValidationError
router = fastapi.APIRouter()
#router.get("/api/weather/{city}")
async def weather(loc: Location = Depends(), units: Optional[str] = "metric"):
return await route_optima(loc.city, loc.state, loc.country, units)
And if I am using the wrong approach here then please point to the best approach to take here.
As you are using Fastapi, it is better to use aioredis to leverage the async functionality.
It turns out I was returning the wrong data it should be like this
# If cache is found then serves the data from cache
if data is not None:
data = data.decode("UTF-8")
data_dict = json.loads(data)
print(data_dict)
print(type(data_dict))
data_dict["cache"] = True
return data_dict
and everything works fine.
I have attached the code. I want chat_id to be accessed from the text file located at my local machine.
#constants.py
chatid_list = []
file1 = open(r'D:\\folder1\\chatids.txt', "r")
for chat_id in file1:
chatid_list.append(chat_id)
#main.py
def start(update, context):
chat_id = update.message.chat_id
first_name =update.message.chat.first_name
last_name = update.message.chat.last_name
username = update.message.chat.username
if chat_id in cp.chatid_list:
print("chat_id : {} and firstname : {} lastname : {} username {}". format(chat_id, first_name, last_name , username))
context.bot.send_message(chat_id, 'Hi ' + first_name + ' Whats up?')
update.message.reply_text( text=main_menu_message(),reply_markup=main_menu_keyboard())
else:
print("WARNING: Unauthorized access denied for {}.".format(chat_id))
update.message.reply_text('User disallowed.')
My guess would be that the elements of chatid_list are strings because you read them from a text file, but chat_id is an integer. So you'd either have to convert chat_id to a string or the elements of chatid_list to integers.
As a cleaner alternative, you could consider using environment variables (e.g. with dotenv) to add configuration elements like the chat IDs (as strings, as noted by #CallMeStag).
That would remove the necessity of having to format and read a file for that purpose. In this case, you can have a .env file in the same directory with:
#.env file
# allowed ids - users allowed to use bot
ALLOWED_IDS = 12345678, 24567896
I'm assuming you're using python 3 and can use f-strings.
So, you can throw your constants.py away and your main.py would look like below:
#main.py file
import os
from dotenv import load_dotenv
load_dotenv()
def start(update, context):
chat_id = update.message.chat_id
first_name = update.message.chat.first_name
last_name = update.message.chat.last_name
username = update.message.chat.username
if chat_id in os.getenv('ALLOWED_IDS'):
print(f'chat_id : {chat_id } and firstname : {first_name } lastname : {last_name } username {username }')
context.bot.send_message(chat_id, f'Hi {first_name}, Whats up?')
update.message.reply_text(text=main_menu_message(),reply_markup=main_menu_keyboard())
else:
print(f'WARNING: Unauthorized access denied for {chat_id}.')
update.message.reply_text('User disallowed.')
Finally, in case you want multiple functions to be "protected", you can use a wrapper as shown here.
Edit: added alternative local file types after OP's comment.
You could also store your allowed users in a JSON (ids.json):
{
"allowed_users": [
{
"name": "john wick",
"id": 1234567
},
{
"name": "rick wick",
"id": 2345738
}
]
}
And then read the allowed IDs as follows:
import json
with open(r'./ids.json', 'r') as in_file:
allowed_users = json.load(in_file)['allowed_users']
allowed_ids = [user['id'] for user in allowed_users]
OR, you could simply drop all the IDs into a text file (ids.txt), with one ID per line:
1234567
2345738
And then read them as follows:
allowed_ids = []
with open(r'./ids.txt', 'r') as in_file:
for row in in_file:
allowed_ids.append(int(row.strip()))
Finally, replace os.getenv('ALLOWED_IDS') from the above code snippet with allowed_ids.
I'm new to flask and in order to refactor an existing route method on a Flask API, i'm looking for the best practice to reduce it and call method inside the route method.
Acutally the route is designed like that :
#qman.route('/add_report/', methods=['POST'])
def create_report():
"""
Check if data send throught http POST request, is correct based on the report
schema and not already recorded in the table report of the DB.
:param: data from POST request
:return: Ok, valide and imported -> 201, Correct but AlreadyKnown -> 208,
InvalideScheme -> 422
"""
jsonData = request.get_json()
reportSchema = ReportSchema()
try:
data = reportSchema.load(jsonData)
except ValidationError as validation_err:
return(validation_err.messages), 422
nameReportCheck = data["report_name"]
report = Report.query.filter_by(report_name=nameReportCheck).first()
if report is None:
# Create new report
report = Report(
report_name=nameReportCheck,
hostname=data["hostname"],
status=data["status"],
date=data["date"],
nb_analysis_erreur=data["nb_analysis_erreur"]
)
db.session.add(report)
db.session.commit()
NewResult = reportSchema.dump(Report.query.get(report.reportID))
return{"message" : "Created new report" , "report" : NewResult}, 201
else :
reportAlreadyKnown = reportSchema.dump(Report.query.get(report.reportID))
return{"message" : "This report is already in the DB", "report" : reportAlreadyKnown}, 208
In the facts i would like to call a function named valid_schema(_schema, _jsondata) to check if the data send throught POST request match with my schema of model Report().
This function return a Response() object with serialized data and a 200 code if it's serialization is possible or an error that i cath inside try/except with 400 error code.
def valid_schema(_schema, _jsondata):
schema = _schema()
try:
data = schema.load(_jsondata)
except ValidationError as validation_err:
response = Response(validation_err.messages, 422)
return response
response = Response(data, 200, mimetype="application/json")
return response
Then the route method call an other function named create_report(report_data) if valid_schema(_schema, _jsondata) return report_data and 200 code in response object.
With his args, this method check if the records is not already in the DB and if is not, he create a Report() object from report_data arg and insert this one as a new record into the DB.
In fact I guess I can easily call this method inside the route function but it seem weird and there is probably an other way that I can't find, maybe decorator ?
One possibility for refactoring is the use of webargs, Flask-Marshmallow and marshmallow-sqlalchemy.
With Flask-Marshmallow you can check the input by specifying fields and validators. Webargs offers you the option of validating the defined scheme in a decorator and passing it on to the route as an argument. Using marshmallow-sqlalchemy in combination, this is immediately converted into a database model.
The following example is based on your information and gives you a brief overview of the usage. By defining your own error handler, the error messages can also be sent as JSON. Use in blueprints, views or the like is possible.
from flask import Flask
from flask import jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow
from marshmallow.validate import Length, OneOf
from webargs.flaskparser import use_args
app = Flask(__name__)
db = SQLAlchemy(app)
ma = Marshmallow(app)
class Report(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String, unique=True)
hostname = db.Column(db.String)
status = db.Column(db.String)
date = db.Column(db.DateTime)
nb_analysis_error = db.Column(db.String)
class ReportSchema(ma.SQLAlchemyAutoSchema):
class Meta:
model = Report
load_instance = True
sqla_session = db.session
name = ma.Str(required=True, validate=Length(min=3))
hostname = ma.Str(required=True)
date = ma.DateTime(required=True)
status = ma.Str(required=True, validate=OneOf(['online', 'offline']))
nb_analysis_error = ma.Str(missing='Unknown Error')
#app.route('/add_report', methods=['POST'])
#use_args(ReportSchema(), location='json')
def add_report(report):
report_schema = ReportSchema()
_report = Report.query.filter_by(name=report.name).first()
if _report:
report_data = report_schema.dump(_report)
return jsonify(message='Already Reported', report=report_data), 208
else:
db.session.add(report)
db.session.commit()
report_data = report_schema.dump(report)
return jsonify(message='Created', report=report_data), 201
with app.app_context():
db.drop_all()
db.create_all()
I want to make a serverless application in AWS Lambda for phone book searches.
What I've done:
Created a bucket and uploaded a CSV file to it.
Created a role with full access to the bucket.
Created a Lambda function
Created API Gateway with GET and POST methods
The Lambda function contains the following code:
import boto3
import json
s3 = boto3.client('s3')
resp = s3.select_object_content(
Bucket='namebbacket',
Key='sample_data.csv',
ExpressionType='SQL',
Expression="SELECT * FROM s3object s where s.\"Name\" = 'Jane'",
InputSerialization = {'CSV': {"FileHeaderInfo": "Use"}, 'CompressionType': 'NONE'},
OutputSerialization = {'CSV': {}},
)
for event in resp['Payload']:
if 'Records' in event:
records = event['Records']['Payload'].decode('utf-8')
print(records)
elif 'Stats' in event:
statsDetails = event['Stats']['Details']
print("Stats details bytesScanned: ")
print(statsDetails['BytesScanned'])
print("Stats details bytesProcessed: ")
print(statsDetails['BytesProcessed'])
print("Stats details bytesReturned: ")
print(statsDetails['BytesReturned'])
When I access the Invoke URL, I get the following error:
{errorMessage = Handler 'lambda_handler' missing on module 'lambda_function', errorType = Runtime.HandlerNotFound}
CSV structure: Name, PhoneNumber, City, Occupation
How to solve this problem?
Please refer to this documentation topic to learn how to write a Lambda function in Python. You are missing the Handler. See: AWS Lambda function handler in Python
Wecome to S.O. #smac2020 links you to the right place AWS Lambda function handler in Python. In short, AWS Lambda needs to know where to find your code, hence the "handler". Though a better way to think about it might be "entry-point."
Here is a close approximation of your function, refactored for use on AWS Lambda:
import json
import boto3
def function_to_be_called(event, context):
# TODO implement
s3 = boto3.client('s3')
resp = s3.select_object_content(
Bucket='stack-exchange',
Key='48836509/dogs.csv',
ExpressionType='SQL',
Expression="SELECT * FROM s3object s where s.\"breen_name\" = 'pug'",
InputSerialization = {'CSV': {"FileHeaderInfo": "Use"}, 'CompressionType': 'NONE'},
OutputSerialization = {'CSV': {}},
)
for event in resp['Payload']:
if 'Records' in event:
records = event['Records']['Payload'].decode('utf-8')
return {
'statusCode': 200,
'body': json.dumps('Hello from Lambda!'),
'pugInfo': records
}
This function produces the following result:
Response
{
"statusCode": 200,
"body": "\"Hello from Lambda!\"",
"currentWorkdingDirectory": "/var/task",
"currentdirlist": [
"lambda_function.py"
],
"pugInfo": "1,pug,toy\r\n"
}
The "entry point" for this function is in a Python file called lambda_function.py and the function function_to_be_called. Together these are the "handler." We can see this in the Console:
or using the API through Boto3
import boto3
awslambda = boto3.client('lambda')
awslambda.get_function_configuration('s3SelectFunction')
Which returns:
{'CodeSha256': 'mFVVlakisUIIsLstQsJUpeBIeww4QhJjl7wJaXqsJ+Q=',
'CodeSize': 565,
'Description': '',
'FunctionArn': 'arn:aws:lambda:us-east-1:***********:function:s3SelectFunction',
'FunctionName': 's3SelectFunction',
'Handler': 'lambda_function.function_to_be_called',
'LastModified': '2021-03-10T00:57:48.651+0000',
'MemorySize': 128,
'ResponseMetadata': ...
'Version': '$LATEST'}
I have a table like this:
class Mapping(db.Model):
map_id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer)
bike_id = db.Column(db.String(255))
is_active_data = db.Column(db.Boolean, default=True)
created_by = db.Column(db.String(150))
updated_by = db.Column(db.String(150))
My POST method:
def save_MapperM(adddata):
create_data = Mapping(**adddata)
db.session.add(create_data)
db.session.commit()
return dict(Successful="Successfully Created")
Route:
#test.route("/addmapper"))
class MapperV(Resource):
#staticmethod
def post():
if request.method == 'POST':
save_data = request.get_json()
try:
return jsonify(save_MapperM(save_data))
except Exception:
return jsonify({'Unsuccessful': 'Looks like you missed something !!'})
Current Code :
The current code will take only one bike_id for every request.
Requirements:
I want to take multiple bike_id's as for one user id and store it as multiple records in the table level.
Example data format coming from UI:
{ user_id: 1, bike_id: 1,2,3,4 }
The easiest solution is to modify your save_MapperM function with a cycle:
def save_MapperM(adddata):
for bike_id in adddata["bike_id"]:
item_data = adddata.copy()
item_data["bike_id"] = bike_id
create_data = Mapping(**item_data)
db.session.add(create_data)
db.session.commit()
return dict(Successful="Successfully Created")
But be careful with this function as it allows to create Mapping instances with any parameters received from the POST request. It looks like it is intended but it can lead to security issues if your Mapping class has some private attributes which should be filled only on the server side. For example the user "Mike" can send a request:
{ "user_id": 1, "bike_id": [1, 2], "created_by": "Alex", "updated_by": "Alex" }
It will cause the save_MapperM function to create instances with created_by and updated_by values set to "Alex" which may not be true. It is better to get such attributes from the session data.
So your post method may look like this (post and save_MapperM functionality combined):
def post():
request_data = request.get_json()
for bike_id in request_data.get("bike_id", []):
item = Mapping(
user_id=request_data.get("user_id"),
bike_id=bike_id,
created_by=session.get("username"),
updated_by=session.get("username"),
)
db.session.add(item)
try:
db.session.commit()
except Exception:
return jsonify({"success": False})
return jsonify({"success": True})
The next step may be implementing request JSON data validation. It is OK when you have a couple of JSON keys with a simple structure but when you need to pass lots of data you need to be sure it is correct. You can use some of the serialization/ODM libraries for this Marshmallow for example.