I'm trying to implement the redis caching in my API for the first time it is a simple fastapi application which is using openweather api to query some weather information and my intention is to cache each json response in the redis server. I'hve made it to work with single key city but with this approach validating query parameters not working with the error handling I put in the place.
caching.py
import sys
from datetime import timedelta
import json
import redis
from services.openweather_service import get_report
def redis_connect() -> redis.client.Redis:
try:
client = redis.Redis(
host="localhost",
port=6379,
db=0,
)
ping = client.ping()
if ping is True:
return client
except redis.ConnectionError:
print("Connection Error!")
sys.exit(1)
client = redis_connect()
def get_routes_from_cache(key: str) -> str:
"""Data from redis."""
val = client.get(key)
return val
def set_routes_to_cache(key: str, value: str) -> bool:
"""Data to redis."""
state = client.setex(
key,
timedelta(hours=24),
value=value,
)
return state
async def route_optima(city: str, state: str, country: str, units=None) -> dict:
location = {"city": city, "state": state, "country": country, "units": units}
# First it looks for the data in redis cache
data = get_routes_from_cache(key=json.dumps(location))
# print(data)
# print(type(data))
# If cache is found then serves the data from cache
if data is not None:
data = data.decode("UTF-8")
data_dict = json.loads(data)
print(data_dict)
print(type(data_dict))
data["cache"] = True
return data
else:
# If cache is not found then sends request to the OpenWeather API
data = await get_report(city, state, country, units)
# This block sets saves the respose to redis and serves it directly
data["cache"] = False
data = json.dumps(data)
state = set_routes_to_cache(key=json.dumps(location), value=json.dumps(data))
if state is True:
return json.dumps(data)
return data
Then I took a different approach by making the query params location = {"city": city, "state": state, "country": country, "units": units} as a key and the json response as a value but when app try to get the response from the cache here it got weird like right after passing the query params dict into the json.dumps it gives me the bytes object and so I decode it with decode("utf-8") but instead of coming back as a dict it gives me <class 'str'> type object. There I am lost... can anyone help me out here?
weather_api.py
from typing import Optional
import fastapi
from fastapi import Depends
from models.location import Location
from infrastructure.caching import route_optima
from models.validation_error import ValidationError
router = fastapi.APIRouter()
#router.get("/api/weather/{city}")
async def weather(loc: Location = Depends(), units: Optional[str] = "metric"):
return await route_optima(loc.city, loc.state, loc.country, units)
And if I am using the wrong approach here then please point to the best approach to take here.
As you are using Fastapi, it is better to use aioredis to leverage the async functionality.
It turns out I was returning the wrong data it should be like this
# If cache is found then serves the data from cache
if data is not None:
data = data.decode("UTF-8")
data_dict = json.loads(data)
print(data_dict)
print(type(data_dict))
data_dict["cache"] = True
return data_dict
and everything works fine.
Related
I've created an Azure Function that fetches data from an External API and sends that data to my cosomos container using a timer trigger. This function is not sending any data to the cosmos db and I can't figure out why.
I've removed some lines of code which contained sensitive information, but I have tested the code locally and it is fetching the data and sending to cosmos if I just run the file.
here is my function:
import datetime
import logging
import azure.functions as func
import time
import pandas as pd
import requests
import json
import uuid
from azure.cosmos.aio import CosmosClient
import asyncio
def get_data():
url = 'externalAPIUrl'
session = requests.Session()
request = session.get(url, headers=headers)
cookies = dict(request.cookies)
response = session.get(url, headers=headers, cookies=cookies).json()
rawData = pd.DataFrame(response)
rawop = pd.DataFrame(rawData['filtered']['data'])
rawop = rawop.set_index('paramter')
processed_data = []
for i in rawop.index:
#creating json to send
processed_data.append(processed_ce)
processed_data.append(processed_pe)
logging.info("Data processed successfully")
return processed_data
async def manage_cosmos(processed_data):
print(len(processed_data))
cosmosdb_endpoint = 'endpointURL'
cosmos_key = 'key'
DATABASE_NAME = 'dbname'
CONTAINER_NAME = 'containername'
async with CosmosClient(cosmosdb_endpoint, cosmos_key) as client:
database = client.get_database_client(DATABASE_NAME)
container = database.get_container_client(CONTAINER_NAME)
# send all the objects in the processed_Data list to cosmos asyncronously
tasks = []
for i in processed_data[:40]:
tasks.append(container.create_item(i))
await asyncio.gather(*tasks)
tasks = []
for i in processed_data[40:90]:
tasks.append(container.create_item(i))
await asyncio.gather(*tasks)
tasks = []
for i in processed_data[90:]:
tasks.append(container.create_item(i))
await asyncio.gather(*tasks)
logging.info("Data sent to cosmos successfully")
def main(mytimer: func.TimerRequest) -> None:
if mytimer.past_due:
logging.info('The timer is past due!')
try:
processed_data = get_data()
asyncio.run(manage_cosmos(processed_data))
except Exception as e:
logging.error("error in function", e)
here is my function.json
{
"scriptFile": "__init__.py",
"bindings": [
{
"name": "mytimer",
"type": "timerTrigger",
"direction": "in",
"schedule": "0 */2 * * * *"
}
]
}
this is my requirements.txt
# DO NOT include azure-functions-worker in this file
# The Python Worker is managed by Azure Functions platform
# Manually managing azure-functions-worker may cause unexpected issues
azure-functions
pandas
asyncio
aiohttp
requests
uuid
azure-cosmos
Can somebody tell me why it is not sending data to cosmos?
I deployed the above function and no data was added to my cosmos container. I ran the same code without the azure-function part locally and I was able to send new items to my cosmos container.
I am most probably making a mistake in creating the function so if anybody can point that out.
I have been working with server sent events to send out certain type of notifications to only certain clients. I am using the module called sse-starlette to try and achieve this.
I am fairly new to FastApi so I am not able to figure out how to send the data to only certain clients instead of broadcasting to everyone.
This is what I have thought so far:
Subscribe request with query param
localhost:8000/subscribe?id=1
from sse_starlette.sse import EventSourceResponse
class EmitEventModel(BaseModel):
event_name: str
event_data: Optional[str] = "No Event Data"
event_id: Optional[int] = None
recipient_id: str
async def connection_established():
yield dict(data="Connection established")
clients = {}
#app.get("/subscribe")
async def loopBackStream(req: Request, id: str = ""):
clients[id] = EventSourceResponse(connection_established())
return clients[id]
#app.post("/emit")
async def emitEvent(event: EmitEventModel):
if clients[event.recipient_id]:
clients[event.recipient_id](publish_event())
Whenever there is an api call to localhost:8000/emit containing the body, Based on the recipient_id the event is going to be routed.
Ofcourse this doesn't work so far. Any pointers as to what should be done to achieve this?
sse_starlette for reference:
https://github.com/sysid/sse-starlette/blob/master/sse_starlette/sse.py
The idea here is that you're going to need to identify the recipient_id on the SSE generator. I've slightly modified your code, to be able to show what I mean:
from __future__ import annotations
import asyncio
import itertools
from collections import defaultdict
from fastapi import Request, FastAPI
from pydantic import BaseModel
from sse_starlette.sse import EventSourceResponse
app = FastAPI()
clients = defaultdict(list)
class EmitEventModel(BaseModel):
event_name: str
event_data: Optional[str] = "No Event Data"
event_id: Optional[int] = None
recipient_id: str
async def retrieve_events(recipient_id: str) -> NoReturn:
yield dict(data="Connection established")
while True:
if recipient_id in clients and len(clients[recipient_id]) > 0:
yield clients[recipient_id].pop()
await asyncio.sleep(1)
print(clients)
#app.get("/subscribe/{recipient_id}")
async def loopBackStream(req: Request, recipient_id: str):
return EventSourceResponse(retrieve_events(recipient_id))
#app.post("/emit")
async def emitEvent(event: EmitEventModel):
clients[event.recipient_id].append(event)
I'm new to flask and in order to refactor an existing route method on a Flask API, i'm looking for the best practice to reduce it and call method inside the route method.
Acutally the route is designed like that :
#qman.route('/add_report/', methods=['POST'])
def create_report():
"""
Check if data send throught http POST request, is correct based on the report
schema and not already recorded in the table report of the DB.
:param: data from POST request
:return: Ok, valide and imported -> 201, Correct but AlreadyKnown -> 208,
InvalideScheme -> 422
"""
jsonData = request.get_json()
reportSchema = ReportSchema()
try:
data = reportSchema.load(jsonData)
except ValidationError as validation_err:
return(validation_err.messages), 422
nameReportCheck = data["report_name"]
report = Report.query.filter_by(report_name=nameReportCheck).first()
if report is None:
# Create new report
report = Report(
report_name=nameReportCheck,
hostname=data["hostname"],
status=data["status"],
date=data["date"],
nb_analysis_erreur=data["nb_analysis_erreur"]
)
db.session.add(report)
db.session.commit()
NewResult = reportSchema.dump(Report.query.get(report.reportID))
return{"message" : "Created new report" , "report" : NewResult}, 201
else :
reportAlreadyKnown = reportSchema.dump(Report.query.get(report.reportID))
return{"message" : "This report is already in the DB", "report" : reportAlreadyKnown}, 208
In the facts i would like to call a function named valid_schema(_schema, _jsondata) to check if the data send throught POST request match with my schema of model Report().
This function return a Response() object with serialized data and a 200 code if it's serialization is possible or an error that i cath inside try/except with 400 error code.
def valid_schema(_schema, _jsondata):
schema = _schema()
try:
data = schema.load(_jsondata)
except ValidationError as validation_err:
response = Response(validation_err.messages, 422)
return response
response = Response(data, 200, mimetype="application/json")
return response
Then the route method call an other function named create_report(report_data) if valid_schema(_schema, _jsondata) return report_data and 200 code in response object.
With his args, this method check if the records is not already in the DB and if is not, he create a Report() object from report_data arg and insert this one as a new record into the DB.
In fact I guess I can easily call this method inside the route function but it seem weird and there is probably an other way that I can't find, maybe decorator ?
One possibility for refactoring is the use of webargs, Flask-Marshmallow and marshmallow-sqlalchemy.
With Flask-Marshmallow you can check the input by specifying fields and validators. Webargs offers you the option of validating the defined scheme in a decorator and passing it on to the route as an argument. Using marshmallow-sqlalchemy in combination, this is immediately converted into a database model.
The following example is based on your information and gives you a brief overview of the usage. By defining your own error handler, the error messages can also be sent as JSON. Use in blueprints, views or the like is possible.
from flask import Flask
from flask import jsonify
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow
from marshmallow.validate import Length, OneOf
from webargs.flaskparser import use_args
app = Flask(__name__)
db = SQLAlchemy(app)
ma = Marshmallow(app)
class Report(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String, unique=True)
hostname = db.Column(db.String)
status = db.Column(db.String)
date = db.Column(db.DateTime)
nb_analysis_error = db.Column(db.String)
class ReportSchema(ma.SQLAlchemyAutoSchema):
class Meta:
model = Report
load_instance = True
sqla_session = db.session
name = ma.Str(required=True, validate=Length(min=3))
hostname = ma.Str(required=True)
date = ma.DateTime(required=True)
status = ma.Str(required=True, validate=OneOf(['online', 'offline']))
nb_analysis_error = ma.Str(missing='Unknown Error')
#app.route('/add_report', methods=['POST'])
#use_args(ReportSchema(), location='json')
def add_report(report):
report_schema = ReportSchema()
_report = Report.query.filter_by(name=report.name).first()
if _report:
report_data = report_schema.dump(_report)
return jsonify(message='Already Reported', report=report_data), 208
else:
db.session.add(report)
db.session.commit()
report_data = report_schema.dump(report)
return jsonify(message='Created', report=report_data), 201
with app.app_context():
db.drop_all()
db.create_all()
I want to terminate rest request coming to server without further processing if input params are missing.
Currently this is the implementation, which I think is not very good for verify_required_params().
I want to terminate this request without returning any value from verify_required_params() in case of missing params. else flow should continue.
Running this on flask server and open to include any new package for best/ optimized approach.
Can please someone suggest an optimize way for this?
#app.route('/is_registered', methods=['POST'])
def is_registered():
_json = request.get_json()
keys = _json.keys()
customer = Customer()
if verify_required_params(['mobile'], keys) is True:
_mobile = _json['mobile']
validated = validate_mobile(_mobile)
registered = customer.find(_mobile)
if not validated:
response = get_response('MOBILE_NUMBER_NOT_VALID')
return jsonify(response)
if not registered:
response = get_response('MOBILE_NUMBER_NOT_REGISTERED')
return jsonify(response)
response = get_response('MOBILE_NUMBER_REGISTERED')
return jsonify(response)
else:
return verify_required_params(['mobile'], keys)
def verify_required_params(required, received):
required = set(required)
received = set(received)
missing = list(sorted(required - received))
data = {"missing_key(s)": missing}
# response = app.response_class(
# response=json.dumps(data),
# status=200,
# mimetype='application/json'
# )
if missing:
return jsonify(data)
return True
🎶 You say it works in a RESTful way, then your errors come back as 200 OK 🎶
In REST, your URL should encode all the information about your entity. In your case, you are identifying a client by their phone number, and you are getting rather than updating information about them, so your endpoint should look like GET /client/<phonenumber>/registered. That way, a request can't not provide this information without going to a different endpoint.
In short, your code will be replaced with:
#app.route('/client/<mobile>/registered', methods=['GET'])
def is_registered(mobile):
if not mobile.is_decimal():
return jsonify({'error': 'mobile is not number'}), 400 # Bad Request
customer = Customer()
registered = bool(customer.find(mobile))
# does it make sense to have a customer who is not registered yet?
# if not, use:
if not registered:
return jsonify({'error': 'client not found'}), 404 # Not Found
validated = validate_mobile(mobile)
return jsonify( {'validated': validated, 'registered': registered} )
In addition, it's better to have the validation function be a decorator. That way it gets called before the actual business logic of the function. For your example of checking whether request.get_json() contains the proper fields, this is how it would look like:
import functools
def requires_fields(fields):
required_fields = set(fields)
def wrapper(func):
#functools.wraps(decorated)
def decorated(*args, **kwargs):
current_fields = set(request.get_json().keys())
missing_fields = required_fields
if missing_fields:
return jsonify({'error': 'missing fields', 'fields': list(missing_fields)}), 400 # Bad Request
resp = func(*args, **kwargs)
return resp
return wrapper
# usage:
#app.route('/comment', methods=['POST'])
#requires_fields(['author', 'post_id', 'body'])
def create_comment():
data = request.get_json()
id = FoobarDB.insert('comment', author=data['author'], post_id=data['post_id'], body=data['body'])
return jsonify({'new_id': id}), 201 # Created
If you must leave it the way it is now, then in order to not return data from the validation function, you must raise an HTTPException. The default function to do it is flask.abort(code).
I have one mongodb database and I have connected that db with pymongo in django. I am new to django, I am trying to get if the entered data present in the collection or not, if present return that record using get method
import pymongo
from pymongo import MongoClient
db_name = 'student_db'
client = MongoClient('localhost', 27017)
db_obj = client[db_name]
collection=db_obj['mongo_app_student']
#api_view(['GET'])
def test(request):
data = request.data
for x in collection.find():
if data in x:
print('entered a right value')
return Response(data)
TypeError at /test
unhashable type: 'dict'
I am getting this error when i am trying to get the output in postman. please help
First you Should use a POST request for that and since find() return a cursor, you're trying to iterate on a cursor. I'm not sure that's a good idea. And assuming request.data is a dict() try using == for comparison with x
Also Try casting what you get from mongo in a list like this :
import pymongo
from pymongo import MongoClient
db_name = 'student_db'
client = MongoClient('localhost', 27017)
db_obj = client[db_name]
collection=db_obj['mongo_app_student']
#api_view(['GET', 'POST'])
def test(request):
response_data = None
if request.method == 'POST':
for x in list(collection.find()):
if data == x:
print('entered a right value')
response_data = data
return Response(response_data )
Let me know how it goes.