I have a python method that lists certain ECS Fargate services based on some tags and some names using boto3 get_paginator.
def list_service_name(environment,
resource_owner_name,
ecs_client):
list_of_service = list()
cluster_name = "my cluster name " + environment
target_string = "-somedummy"
resource_owner_tag = resource_owner_name
service_paginator = ecs_client.get_paginator('list_services')
for page in service_paginator.paginate(cluster=cluster_name,
launchType='FARGATE'):
for service in page['serviceArns']:
response = ecs_client.list_tags_for_resource(resourceArn=service)
for tags in response['tags']:
if tags['key'] == 'ResourceOwner' and \
tags['value'] == resource_owner_tag and \
service.endswith(target_string):
list_of_service.append(service)
return list_of_service
Now I would like to test this using moto.
Hence I have created conftest.py where I have defined all moto mock connections to the services like, ecs. Also, I have created test_main.py file like below where I have created dummy services connected to ECS Fargate. But for some reason, if I try to assert the outcome of the main method in the test file the service lists return empty. Whereas I would like to see test-service-for-successful as the outcome. Is there something I'm missing or pagination is still not available in moto?
from my_code.main import *
#pytest.fixture
def env_name():
return "local"
#pytest.fixture
def cluster_name(env_name):
return "my dummy" + env_name + "cluster_name"
#pytest.fixture
def successful_service_name():
return "test-service-for-successful"
#pytest.fixture
def un_successful_service_name():
return "test-service-for-un-successful"
#pytest.fixture
def resource_owner():
return "dummy_tag"
#pytest.fixture
def test_create_service(ecs_client,
cluster_name,
successful_service_name,
un_successful_service_name,
resource_owner):
_ = ecs_client.create_cluster(clusterName=cluster_name)
_ = ecs_client.register_task_definition(
family="test_ecs_task",
containerDefinitions=[
{
"name": "hello_world",
"image": "docker/hello-world:latest",
"cpu": 1024,
"memory": 400,
"essential": True,
"environment": [
{"name": "environment", "value": "local"}
],
"logConfiguration": {"logDriver": "json-file"},
}
],
)
ecs_client.create_service(
cluster=cluster_name,
serviceName=successful_service_name,
taskDefinition="test_ecs_task",
desiredCount=0,
launchType="FARGATE",
tags=[{"key": "resource_owner", "value": resource_owner}]
)
ecs_client.create_service(
cluster=cluster_name,
serviceName=un_successful_service_name,
taskDefinition="test_ecs_task",
desiredCount=0,
launchType="FARGATE",
tags=[{"key": "resource_owner", "value": resource_owner}]
)
yield
def test_list_service_name(env_name,
resource_owner,
ecs_client):
objects = list_service_name(env_name,
resource_owner,
ecs_client)
# here object is []
# Where as I should see successful_service_name
Related
I am using pytest and moto3 to test some code similar to this:
response = athena_client.start_query_execution(
QueryString='SELECT * FROM xyz',
QueryExecutionContext={'Database': myDb},
ResultConfiguration={'OutputLocation': someLocation},
WorkGroup=myWG
)
execution_id = response['QueryExecutionId']
if response['QueryExecution']['Status']['State'] == 'SUCCEEDED':
response = athena_client.get_query_results(
QueryExecutionId=execution_id
)
results = response['ResultSet']['Rows']
...etc
In my test I need that the values from results = response['ResultSet']['Rows'] are controlled by the test. I am using some code like this:
backend = athena_backends[DEFAULT_ACCOUNT_ID]["us-east-1"]
rows = [{"Data": [{"VarCharValue": "xyz"}]}, {"Data": [{"VarCharValue": ...}, etc]}]
column_info = [
{
"CatalogName": "string",
"SchemaName": "string",
"TableName": "xyz",
"Name": "string",
"Label": "string",
"Type": "string",
"Precision": 123,
"Scale": 123,
"Nullable": "NOT_NULL",
"CaseSensitive": True,
}
]
results = QueryResults(rows=rows, column_info=column_info)
backend.query_results[NEEDED_QUERY_EXECUTION_ID] = results
but that is not working as I guess NEEDED_QUERY_EXECUTION_ID is not known before from the test. How can I control it?
UPDATE
Based on suggestion I tried to use:
results = QueryResults(rows=rows, column_info=column_info)
d = defaultdict(lambda: results.to_dict())
backend.query_results = d
to force a return of values, but it seems not working as from the moto3's models.AthenaBackend.get_query_results, I have this code:
results = (
self.query_results[exec_id]
if exec_id in self.query_results
else QueryResults(rows=[], column_info=[])
)
return results
which will fail as the if condition won't be satifsfied.
Extending the solution of the defaultdict, you could create a custom dictionary that contains all execution_ids, and always returns the same object:
class QueryDict(dict):
def __contains__(self, item):
return True
def __getitem__(self, item):
rows = [{"Data": [{"VarCharValue": "xyz"}]}, {"Data": [{"VarCharValue": "..."}]}]
column_info = [
{
"CatalogName": "string",
"SchemaName": "string",
"TableName": "xyz",
"Name": "string",
"Label": "string",
"Type": "string",
"Precision": 123,
"Scale": 123,
"Nullable": "NOT_NULL",
"CaseSensitive": True,
}
]
return QueryResults(rows=rows, column_info=column_info)
backend = athena_backends[DEFAULT_ACCOUNT_ID]["us-east-1"]
backend.query_results = QueryDict()
An alternative solution to using custom dictionaries would to be seed Moto.
Seeding Moto ensures that it will always generate the same 'random' identifiers, which means you always know what the value of NEEDED_QUERY_EXECUTION_ID is going to be.
backend = athena_backends[DEFAULT_ACCOUNT_ID]["us-east-1"]
rows = [{"Data": [{"VarCharValue": "xyz"}]}, {"Data": [{"VarCharValue": "..."}]}]
column_info = [...]
results = QueryResults(rows=rows, column_info=column_info)
backend.query_results["bdd640fb-0667-4ad1-9c80-317fa3b1799d"] = results
import requests
requests.post("http://motoapi.amazonaws.com/moto-api/seed?a=42")
# Test - the execution id will always be the same because we just seeded Moto
execution_id = athena_client.start_query_execution(...)
Documentation on seeding Moto can be found here: http://docs.getmoto.org/en/latest/docs/configuration/recorder/index.html#deterministic-identifiers
(It only talks about seeding Moto in the context of recording/replaying requests, but the functionality can be used on it's own.)
I run the following signals.py on (Ubuntu2204/WSL2 Windows 11) using Django 4.1.1 / Python 3.10
from .models import Sale
from django.db.models.signals import m2m_changed
from django.dispatch import receiver
#receiver(m2m_changed, sender=Sale.positions.through)
def calculate_total_price(sender, instance, action, **kwargs):
print('action', action)
total_price = 0
if action == 'post_add' or action == 'post_remove':
for item in instance.get_positions():
total_price += item.price
instance.total_price = total_price
instance.save()
apps.py already signals in VSCode that signals is not used, this is also confirmed in the Django Debug Toolbar
from django.apps import AppConfig
class SalesConfig(AppConfig):
#default_auto_field = 'django.db.models.BigAutoField'
name = 'sales'
def ready(self):
import sales.signals
and then the __init__.py file
default_app_config = 'sales.apps.SalesConfig'
settings.py looks like this:
"""
Django settings for reports_proj project.
Generated by 'django-admin startproject' using Django 4.1.1.
For more information on this file, see
https://docs.djangoproject.com/en/4.1/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.1/ref/settings/
"""
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "django-insecure-dbl^hk-m6-dz01+1i*hi5#rz4t90(y5s-cls1&0js5hj^ojo70"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"debug_toolbar",
# my apps
"customers",
"products",
"profiles",
"reports",
"sales.apps.SalesConfig",
# 3rd party
"crispy_forms",
]
CRISPY_TEMPLATE_PACK = "bootstrap4"
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"debug_toolbar.middleware.DebugToolbarMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "reports_proj.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [BASE_DIR / 'templates'],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
WSGI_APPLICATION = "reports_proj.wsgi.application"
# Database
# https://docs.djangoproject.com/en/4.1/ref/settings/#databases
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": BASE_DIR / "db.sqlite3",
}
}
# Password validation
# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
# Internationalization
# https://docs.djangoproject.com/en/4.1/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.1/howto/static-files/
STATIC_URL = '/static/'
STATICFILES_DIRS = [
BASE_DIR / 'static',
BASE_DIR / 'sales' / 'static',
]
MEDIA_URL = '/media/'
MEDIA_ROOT = BASE_DIR / 'media'
# Default primary key field type
# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
INTERNAL_IPS = [
"127.0.0.1",
]
Any hint would be highly appreciated, the code looks ok to me it must be some dependency issue but I have no idea where to start looking really
I just noticed that your ready identation is not correct, causing the problem...
You should change this:
from django.apps import AppConfig
class SalesConfig(AppConfig):
#default_auto_field = 'django.db.models.BigAutoField'
name = 'sales'
def ready(self):
import sales.signals
To this:
from django.apps import AppConfig
class SalesConfig(AppConfig):
#default_auto_field = 'django.db.models.BigAutoField'
name = 'sales'
def ready(self):
from . import signals
Generally because no indentation the method became global function and it was not called as it should.
I used WebsocketApp to create a python Websocket client(synch) which interacts with Websocket server.
Now, I am interested in integrating the Websocket client in end to end test.
I tried to write a Pytest for Websocket client, but because I am a beginner with Pytest, so I blocked and I do not know how should I continue it.
this is my code:
import json
import websocket
no_message = 0
count = 1
def on_error(ws, error):
print(error)
def on_close(ws, close_status_code, close_msg):
print("############## closed ###############")
def on_message(ws, message):
try:
if message is not None:
print("RESPONSE FROM SERVER.......", message)
global no_message
no_message += 1
if no_message >= count:
ws.close()
if message is None:
ws.close()
except:
print("error")
ws.on_error()
def on_open(ws):
global count
message = {
"session_id": "String",
"destination": "127.0.0.1",
"count": count,
"command": "LIVE_PING",
}
count = message.get("count")
ws.send(json.dumps(message))
if __name__ == "__main__":
websocket.enableTrace(True)
host = "ws://127.0.0.1:8000/"
ws = websocket.WebSocketApp(
host,
on_open=on_open,
on_message=on_message,
on_error=on_error,
on_close=on_close,
)
ws.run_forever()
and this my pytest for:
import json
import unittest.mock as mock
import websocket
class Testws():
def test_on_close(self):
print("Close")
def test_on_error(self):
print('ERROR')
def test_on_message(self, message):
assert message == 'RESPONSE FROM SERVER....... {"session_id":"String","sample":0,"elapsed":15}'
def test_on_open(self, ws):
fake_message = {
"session_id": "String",
"destination": "127.0.0.1",
"count": 2,
"command": "LIVE_PING",
}
ws.send(json.dumps(fake_message))
c = Testws()
host = "ws://127.0.0.1:8000/"
ws = websocket.WebSocketApp(host, on_open=c.test_on_open, on_message=c.test_on_message, )
ws.run_forever()
I tried also using Monkeypatching, but I am confused and do not think it is the right way
import json
import websocket
from wsClient import synch_ws_client as cl
from websocket import WebSocketApp
def test_synch_ws_client(monkeypatch):
def mock_on_message():
print('RESPONSE FROM SERVER....... {"session_id":"String","sample":0,"elapsed":2}')
def mock_on_error():
print('Error')
def mock_on_close():
print('############## closed ###############')
def mock_on_open(ws):
fake_massage = {
"session_id": "String",
"destination": "127.0.0.1",
"count": 1,
"command": "LIVE_PING"
}
ws.send(json.dumps(fake_massage))
monkeypatch.setattr(WebSocketApp, 'on_message', mock_on_message)
monkeypatch.setattr(WebSocketApp, 'on_close', mock_on_close)
monkeypatch.setattr(WebSocketApp, 'on_error', mock_on_error)
monkeypatch.setattr(WebSocketApp, 'on_open', mock_on_open)
assert cl.on_message() == 'RESPONSE FROM SERVER....... {"session_id":"String","sample":0,"elapsed":2}'
I do not know, how can I check for example the test_on_open and test_on_message modules are call successfully.
Set the stain like this:
v3.patch_node('nodename',
{"spec": {"taints": [{"effect": "NoSchedule", "key": "test", "value": "1",'tolerationSeconds': '300'}]}}```
however.
How to remove tains?
This was pretty non-intuitive to me, but here's how I accomplished this.
def taint_node(context, node_name):
kube_client = setup_kube_client(context)
taint_patch = {"spec": {"taints": [{"effect": "NoSchedule", "key": "test", "value": "True"}]}}
return kube_client.patch_node(node_name, taint_patch)
def untaint_node(context, node_name):
kube_client = setup_kube_client(context)
remove_taint_patch = {"spec": {"taints": []}}
return kube_client.patch_node(node_name, remove_taint_patch)
That worked for me, but it removes ALL taints, which is maybe not what you want to do.
I tried the following:
def untaint_node(context, node_name):
kube_client = setup_kube_client(context)
remove_taint_patch = {"spec": {"taints": [{"effect": "NoSchedule-", "key": "test", "value": "True"}]}}
return kube_client.patch_node(node_name, remove_taint_patch)
but encountered server side validation preventing it (because the effect isn't in the collection of supported values):
kubernetes.client.exceptions.ApiException: (422)
Reason: Unprocessable Entity
HTTP response headers: HTTPHeaderDict({'Audit-Id': 'bfbad6e1-f37c-4090-898b-b2b9c5500425', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'X-Kubernetes-Pf-Flowschema-Uid': '7c028f53-f0a4-46bd-b400-68641158da78', 'X-Kubernetes-Pf-Prioritylevel-Uid': 'ef92e801-ce03-4abb-a607-20921bf82547', 'Date': 'Sat, 18 Sep 2021 02:45:37 GMT', 'Content-Length': '759'})
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Node \"aks-agentpool-33938206-vmss000000\" is invalid: metadata.taints[0].effect: Unsupported value: \"NoSchedule-\": supported values: \"NoSchedule\", \"PreferNoSchedule\", \"NoExecute\"","reason":"Invalid","details":{"name":"aks-agentpool-33938206-vmss000000","kind":"Node","causes":[{"reason":"FieldValueNotSupported","message":"Unsupported value: \"NoSchedule-\": supported values: \"NoSchedule\", \"PreferNoSchedule\", \"NoExecute\"","field":"metadata.taints[0].effect"},{"reason":"FieldValueNotSupported","message":"Unsupported value: \"NoSchedule-\": supported values: \"NoSchedule\", \"PreferNoSchedule\", \"NoExecute\"","field":"metadata.taints[0].effect"}]},"code":422}
Finally, if you need to remove a specific taint, you can always shell out to kubectl (though that's kinda cheating, huh?):
def untaint_node_with_cmd(context, node_name):
cmd_env = os.environ.copy()
child = subprocess.Popen(['kubectl', 'taint', 'nodes', node_name, 'test=True:NoSchedule-', '--context', context], env=cmd_env)
exit_code = child.wait()
return exit_code
Sadly, it doesn't look like this issue has gotten much love in the k8s python client repo. https://github.com/kubernetes-client/python/issues/161
One more better way to untainted a particular taint. By doing this way other taints will not get removed.only a particular taint will ve untainted.
def untaint_node(context, node_name, taint_key):
Kube_client = setup_kube_client(context)
node = Kube_client.list_nodes(field_selector={"metadata.name" : node_name}).items[0]
taints = node.spec.taints
filtered_taints = list(filter(lambda x: x.key != taint_key, taints))
body = {"spec": {"taints": filtered_taints}}
return kube_client.patch_node(node_name, body)
There's nothing special, standard update or patch call on the Node object.
Client libraries are used to interact with kubeapiserver. Therefore, kubeapiserver checks body of the request, no need to have custom removing taint in Python client library.
I think you can do it by calling
v3.patch_node('cn-shanghai.10.10.10.249',
{"spec": {"taints": [{"effect": "NoSchedule-", "key": "test", "value": "1","tolerationSeconds": "300"}]}}
An example can be found in python-client examples repository.
from kubernetes import client, config
def main():
config.load_kube_config()
api_instance = client.CoreV1Api()
body = {
"metadata": {
"labels": {
"foo": "bar",
"baz": None}
}
}
# Listing the cluster nodes
node_list = api_instance.list_node()
print("%s\t\t%s" % ("NAME", "LABELS"))
# Patching the node labels
for node in node_list.items:
api_response = api_instance.patch_node(node.metadata.name, body)
print("%s\t%s" % (node.metadata.name, node.metadata.labels))
if __name__ == '__main__':
main()
Reference: https://github.com/kubernetes-client/python/blob/c3f1a1c61efc608a4fe7f103ed103582c77bc30a/examples/node_labels.py
I have a basic API setup to do a basic Post and Get from a single table. I want to create a nested array though grouping by force_element_type
model.py
from db import db
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy import text as sa_text
class ForceElementModel(db.Model):
__tablename__ = 'force_element'
__table_args__ = {'schema': 'force_element'}
force_element_id = db.Column(UUID(as_uuid=True), primary_key=True, server_default=sa_text("uuid_generate_v4()"))
name = db.Column(db.String(100), nullable=False)
force_element_type = db.Column(db.String(20), nullable=False)
def __init__(self, name, force_element_type):
self.name = name
self.force_element_type = force_element_type
def json(self):
return {'name': self.name, 'force_element_type': self.force_element_type}
#classmethod
def find_by_name(cls, name):
return cls.query.filter_by(name=name).first() # simple TOP 1 select
def save_to_db(self): # Upserting data
db.session.add(self)
db.session.commit() # Balla
def delete_from_db(self):
db.session.delete(self)
db.session.commit()
resource.py
from flask_restful import Resource, reqparse
#from flask_jwt import jwt_required
from models.force_element import ForceElementModel
class ForceElement(Resource):
parser = reqparse.RequestParser() # only allow price changes, no name changes allowed
parser.add_argument('force_element_type', type=str, required=True, help='This field cannot be left blank')
##jwt_required()
def post(self, name):
if ForceElementModel.find_by_name(name):
return {'message': "An Force Element with name '{}' already exists.".format(name)}, 400
data = ForceElement.parser.parse_args()
force_element = ForceElementModel(name, data['force_element_type'])
try:
force_element.save_to_db()
except:
return {"message": "An error occurred inserting the item."}, 500
return force_element.json(), 201
class ForceElementList(Resource):
##jwt_required()
def get(self):
return {'force_elements': [force_element.json() for force_element in ForceElementModel.query.all()]}
class ForceElementType(Resource):
##jwt_required()
def get(self):
The GET endpoint using ForceElementList returns
{
"force_elements": [
{
"name": "San Antonio",
"force_element_type": "ship"
},
{
"name": "Nimitz",
"force_element_type": "ship"
},
{
"name": "Nimitz- Starboard",
"force_element_type": "Crew"
},
{
"name": "Nimitz- Port",
"force_element_type": "Crew"
}
]
}
I don't know how to group by force_element_type and return
[
"ship": [
{
"name": "San Antonio",
"force_element_id": "xxx1"
},
{
"name": "Nimitz",
"force_element_id": "xxx2"
}],
"crew": [
{
"name": "Nimitz- Starboard",
"force_element_id": "yyy1"
},
{
"name": "Nimitz- Port",
"force_element_id": "yyy2"
}
]
]
How do I create this separate andpoint?
OK I got there, here is how I did it. Is there a better way?
Lesson one use an online parser to check the json format this is what I was actually aiming for and the square braket at then start had me scratching my head for a while
{
"ship": [
{
"name": "San Antonio",
"force_element_id": "xxx1"
},
{
"name": "Nimitz",
"force_element_id": "xxx2"
}],
"crew": [
{
"name": "Nimitz- Starboard",
"force_element_id": "yyy1"
},
{
"name": "Nimitz- Port",
"force_element_id": "yyy2"
}]
}
This code creates the correct format for the output
class ForceElementType(Resource):
##jwt_required()
def get(self):
types = {}
force_elements = ForceElementModel.query.order_by(ForceElementModel.force_element_type.desc()).all()
for force_element in force_elements:
nested = {'name': force_element.name, 'force_element_id': str(force_element.force_element_id)}
print(nested)
if not force_element.force_element_type in types:
types[force_element.force_element_type] = []
types[force_element.force_element_type].append(nested)
response = types