Hello currently found a problem which I could not find solution to it.
I have a django application which communicates with external services in order to write unittests i must patch the functions which call the external services with mock.patch
#api_views.py
from examplemodule import examplefunction # just example
class GetSomeInfoFromExternalService(ApiView):
def get(self, *args, **kwargs):
if example_function(attrs) == somevalue:
return Response({'detail':'OK'})
return Response({'detail':'Not ok'})
here is my other file
#tests.py
from unittests import mock
from django.test import TestCase
class MyTestCase(TestCase):
#mock.pach('examplemodule.examplefunction')
def test_1(self):
examplefunction.return_value=123
Like that the patch method will not work, but if I import the examplefunction inside the ApiView.get method is overriding and the mock works.
#api_views.py
class GetSomeInfoFromExternalService(ApiView):
def get(self, *args, **kwargs):
from examplemodule import examplefunction # If the import is here the override is working properly
if example_function(attrs) == somevalue:
return Response({'detail':'OK'})
return Response({'detail':'Not ok'})
I am trying to read data from a kafka topiv which has been serialized using google's protobuf.
I compiled the proto files using protoc which generated pb2 files.
Now i am trying to use faust and create a stream processor but i can't find the correct way to use the pb2 files as key_serializer and value_serializer.
Here is what i have tried:
import faust
from proto.topic_pb2 import topic
app = faust.App(
'faust-consumer',
broker='kafka://',
store="memory://",
cache="memory://",
)
schema = faust.Schema(
## key_type=topic.PK,
## value_type=topic,
key_serializer=topic.PK,
value_serializer=topic,
)
topic = app.topic(
'topic',
schema=schema
)
#app.agent(topic)
async def consume(topic):
async for event in topic:
print(event)
if __name__ == "__main__":
app.main()
Does anybody have any idea how to used the pb2 in the serializers?
Man, I was trying to do the same the past week. After struggling I finally got something working - not the best way - but it works well enough.
So initially I used this python compiler: https://github.com/danielgtaylor/python-betterproto to generate the *.py files with dataclasses / type hinting.
Then, I was able to create Faust.Record classes dynamically by using a helper:
import abc
import inspect
from typing import Type
import betterproto
import faust
GENERATED_SUFFIX = "__FaustRecord_Auto"
def _import_relative_class(module: str, klass_name: str):
resolved_import = __import__(module, fromlist=[klass_name])
klass = getattr(resolved_import, klass_name)
return klass
def _is_record(attype: Type):
return (
inspect.isclass(attype)
and isinstance(attype, betterproto.Message)
or isinstance(attype, abc.ABCMeta)
)
def _build_record_annotations(klass: Type):
annotations = {}
for atname, attype in klass.__annotations__.items():
if _is_record(attype):
annotations[atname] = make_faust_record(attype)
elif isinstance(attype, str):
subklass = _import_relative_class(klass.__module__, attype)
annotations[atname] = make_faust_record(subklass)
else:
annotations[atname] = attype
return annotations
def make_faust_record(klass: Type):
type_name = f"{klass.__name__}{GENERATED_SUFFIX}"
record_type = type(type_name, (faust.Record, klass), {})
record_type.__annotations__ = _build_record_annotations(klass)
record_type._init_subclass()
return record_type
Now you can use it like:
import faust
from proto.your_models import YourModel # Import your generated proto here
from faust_converter import make_faust_record
app = faust.App(
'faust-consumer',
broker='kafka://',
store="memory://",
cache="memory://",
)
model_record = make_faust_record(YourModel)
topic = app.topic(
'topic',
value_type=model_record
)
#app.agent(topic)
async def consume(topic):
async for event in topic:
print(event)
if __name__ == "__main__":
app.main()
I was also experimenting with using Protobuf with Faust.
Mentioned below is the solution using Faust Serialiser Codecs.
faust-protobuf https://github.com/hemantkashniyal/faust-protobuf
proto_serializer.py
from faust.serializers import codecs
from typing import Any
from google.protobuf import json_format
from google.protobuf.json_format import MessageToJson
from google.protobuf.json_format import MessageToDict
from google.protobuf import text_format
from google.protobuf.text_format import MessageToString
from google.protobuf.text_format import MessageToBytes
class ProtobufSerializer(codecs.Codec):
def __init__(self, pb_type: Any):
self.pb_type = pb_type
super(self.__class__, self).__init__()
def _dumps(self, pb: Any) -> bytes:
return pb.SerializeToString()
def _loads(self, s: bytes) -> Any:
pb = self.pb_type()
pb.ParseFromString(s)
return pb
app.py
import faust
from google.protobuf.json_format import MessageToJson
from .proto.greetings_pb2 import Greeting
from .proto_serializer import ProtobufSerializer
app = faust.App(
'faust-consumer',
broker='kafka://', # TODO: update kafka endpoint
store="memory://",
cache="memory://",
)
greetings_schema = faust.Schema(
key_serializer=ProtobufSerializer(pb_type=Greeting),
value_serializer=ProtobufSerializer(pb_type=Greeting),
)
topic = app.topic(
'greetings',
schema=greetings_schema
)
#app.agent(topic)
async def consume(topic):
async for event in topic:
print(MessageToJson(event))
#app.timer(5)
async def produce():
for i in range(10):
data = Greeting(hello="world", message=i)
await consume.send(value=data)
if __name__ == "__main__":
app.main()
I was able to do it by creating a Serializer class as so:
import faust
from abc import ABCMeta, abstractmethod
from google.protobuf.json_format import MessageToDict
from faust.serializers.codecs import Codec
from importlib import import_module
def get_proto(topic_name, only_pk=False):
if not hasattr(get_proto, "topics"):
setattr(get_proto, "topics", dict())
get_proto.topics[topic_name] = import_module(
"protodef.{}_pb2".format(topic_name)
).__getattribute__(topic_name.split(".")[-1])
if only_pk:
return getattr(get_proto, "topics").get(topic_name).PK
else:
return getattr(get_proto, "topics").get(topic_name)
class ProtoSerializer(Codec, metaclass=ABCMeta):
#abstractmethod
def only_key(self):
...
def as_proto(self, topic_name):
self._proto = get_proto(topic_name, self.only_key())
return self
def _loads(self, b):
data = MessageToDict(
self._proto.FromString(b),
preserving_proto_field_name=True,
including_default_value_fields=True,
)
# remove the key object from the unserialized message
data.pop("key", None)
return data
def _dumps(self, o):
# for deletes
if not o:
return None
obj = self._proto()
# add the key object to them message before serializing
if hasattr(obj, "PK"):
for k in obj.PK.DESCRIPTOR.fields_by_name.keys():
if k not in o:
raise Exception(
"Invalid object `{}` for proto `{}`".format(o, self._proto)
)
setattr(obj.key, k, o[k])
for k, v in o.items():
if hasattr(obj, k):
setattr(obj, k, v)
else:
ghost.debug(
"Invalid value-attribute `%s` for proto `%s`", k, self._proto
)
return obj.SerializeToString()
class ProtoValue(ProtoSerializer):
def only_key(self):
return False
class ProtoKey(ProtoSerializer):
def only_key(self):
return True
and then use it as follows:
import faust
from utils.serializer import ProtoKey, ProtoValue
app = faust.App(
'faust-consumer',
broker='kafka://',
store="memory://",
cache="memory://",
)
topic = app.topic(
'topic',
key_serializer=ProtoKey().as_proto('topic'),
value_serializer=ProtoValue().as_proto('topic')
)
#app.agent(topic)
async def consume(topic):
async for event in topic:
print(event)
if __name__ == "__main__":
app.main()
I am trying to do the following:
#patch('uuid.uuid4', autospec=True)
def test_generate_adid(self, patched_uuid, app_api):
patched_uuid.return_value = "9e51ab81-6d65-4b81-af3b-8f7f49d69ba7"
adid = app_api.generate_adid()
assert adid == "9e51ab81-6d65-4b81-af3b-8f7f49d69ba7"
Where app_api is a fixture of the class under test.
However, in my app_api class, uuid4() is not getting patched and keeps returning a uuid other than the one I am trying to force. Here is what the generate_adid() instance method looks like:
from uuid import uuid4
def generate_adid(self):
adid = str(uuid4())
return adid
The failing unit test error:
AssertionError: assert '90b29e86-e3b0-40aa-8971-f868f90cb009' == '9e51ab81-6d65-4b81-af3b-8f7f49d69ba7'
I have consulted this post: How to mock uuid generation in a test case? but still am having no luck.
What am I doing wrong? Thanks to all of those who reply in advance.
EDIT: Here is the full code:
from requests import Session
from random import uniform
from hashlib import md5
from hmac import new
from uuid import uuid4
from json import dumps
class AppApi:
def __init__(self, account):
self.account = account
self.session = Session()
def generate_adid(self):
adid = str(uuid4())
return adid
Test Case:
from src import AppApi
from pytest import fixture
from unittest.mock import patch
from json import loads
ACCOUNT = {
"email": "user#email.com",
"username": "user",
"password": "s3cr3t"
}
#fixture
def app_api():
app_api = AppApi(ACCOUNT)
yield app_api
class TestAppApi:
#patch('uuid.uuid4')
def test_generate_adid(self, patched_uuid, app_api):
patched_uuid.return_value = "9e51ab81-6d65-4b81-af3b-8f7f49d69ba7"
adid = app_api.generate_adid()
assert adid == "9e51ab81-6d65-4b81-af3b-8f7f49d69ba7"
In your example you're patching the uuid4() function in the uuid module rather than the function uuid4() in the module which you're trying to test. Take a look at Python unnit.test docs where to patch
Using your example above you need to patch the uuid4() imported into the src module. You need to use #patch("src.uuid4")
from src import AppApi
from pytest import fixture
from unittest.mock import patch
from json import loads
ACCOUNT = {
"email": "user#email.com",
"username": "user",
"password": "s3cr3t"
}
#fixture
def app_api():
app_api = AppApi(ACCOUNT)
yield app_api
class TestAppApi:
#patch('src.uuid4')
def test_generate_adid(self, patched_uuid, app_api):
patched_uuid.return_value = "9e51ab81-6d65-4b81-af3b-8f7f49d69ba7"
adid = app_api.generate_adid()
assert adid == "9e51ab81-6d65-4b81-af3b-8f7f49d69ba7"
Hope this helps!
I have following file and i am running pytests on this file. One of the use case is to test the logger function which is global. How can we mock this in my pytest ?
import spark_fn
def run_job(params):
do_something()
logger.info("some info")
def main(params):
global logger
app_name = "test app"
spark, logger = spark_fn(app_name)
run_job(params)
if __name__ == "__main__":
params = "some params"
main(params)
===============================================================
How's something like this? You should think of it less like a global variable and more like patching what spark_fn is returning to have a "fake" logger class. Also I'm curious how you're using the spark_fn module overall with how you're importing it.
import yoursampletest
def test_monkeytest(monkeypatch):
"""
Monkey patches a fake logger into spark_fn
"""
class myFakeLogger:
def info(self, value):
return value
def mock_info(mylogstatement):
return 1, myFakeLogger()
monkeypatch.setattr(yoursampletest, 'spark_fn', mock_info)
assert yoursampletest.main(1) == 'some info'
From Intermediate-Usage Flask-RESTful 0.3.7 documentation
in the Passing Constructor Parameters Into Resources section at the bottom, how would you write a test in order to mock kwargs? Side note: I tweaked it so the Smart Engine class is passed directly rather than being instantiated to a variable then passed.
from flask_restful import Resource
class TodoNext(Resource):
def __init__(self, **kwargs):
# smart_engine is a black box dependency
self.smart_engine = kwargs['smart_engine']
def get(self):
return self.smart_engine.next_todo()
You can inject the required dependency into TodoNext like so:
api.add_resource(TodoNext, '/next',
resource_class_kwargs={ 'smart_engine': SmartEngine() })
Test class in question:
import unittest
class TestTodoNext(unittest.TestCase):
todo_next_instance = TodoNext() # How would you mock smart_engine in this case?
You can use Mock object from unittest.mock to mock smart_engine.
import unittest
from unittest.mock import Mock
class TestTodoNext(unittest.TestCase):
smart_engine = Mock()
smart_engine.next_todo.return_value = "YOUR DESIRED RETURN VALUE"
todo_next_instance = TodoNext(smart_engine=smart_engine)
self.assertEqual(todo_next_instace.get(), "YOUR DESIRED RETURN VALUE")