I have python script that queries OCI Compute Instances with a defined tag and value of test and will take the results and stop any instance with those tags. I am expecting the query to return the result as a json object of according to this:
https://docs.oracle.com/en-us/iaas/Content/Search/Tasks/queryingresources.htm
However whenever I take the result and apply it to a variable I get the following error:
"Error 'ResourceSummaryCollection' object is not subscriptable"
I noticed that the query response is not returning as a json object. It is returning as a list.
Update:
This is the error I am getting after updating my script: "Error 'list' object has no attribute 'identifier'". I was expecting the query to return as a json object. When I try to convert the list into a json object via the json dumps method I get object is not serializable. Bottom line is, how do I pass the OCID "Identifier" from the query?
My function:
def do(signer):
print("Searching for untagged instance", flush=True)
# results = ""
# message = ""
# resp = ""
try:
search_client = oci.resource_search.ResourceSearchClient(config={}, signer=signer)
print("Search client initialized",flush=True)
PredefinedTag = "apps"
key= "test"
value= "test"
structured_search = oci.resource_search.models.StructuredSearchDetails(
query="query instance resources where (definedTags.namespace = '{}' && definedTags.key = '{}' && definedTags.value = '{}')".format(PredefinedTag,key,value),
type='Structured',
matching_context_type=oci.resource_search.models.SearchDetails.MATCHING_CONTEXT_TYPE_NONE)
print("Step1",flush=True)
#old
results = search_client.search_resources(structured_search).data
# print(results.items)
# print(results.items.identifier)
# print(results['items'][0]['identifier'])
print("Step2",flush=True)
instanceId = results(results.items.identifier)
# instanceId = results['items'][0]['identifier']
print("Step3",flush=True)
resp = perform_action(signer, instanceId , 'STOP')
print("Step4",flush=True)
except oci.exceptions.ServiceError as e:
print('RQS Search failed with Service Error: {0}'.format(e),flush=True)
raise
except oci.exceptions.RequestException as e:
print('RQS Search failed w/ a Request exception. {0}'.format(e),flush=True)
raise
return resp
Was able to reference the response from the query:
def do(signer):
print("Searching for untagged instance", flush=True)
try:
search_client = oci.resource_search.ResourceSearchClient(config={}, signer=signer)
print("Search client initialized",flush=True)
PredefinedTag = "apps"
key= "test"
value= "test"
structured_search = oci.resource_search.models.StructuredSearchDetails(
query="query instance resources where (definedTags.namespace = '{}' && definedTags.key = '{}' && definedTags.value = '{}')".format(PredefinedTag,key,value),
type='Structured',
matching_context_type=oci.resource_search.models.SearchDetails.MATCHING_CONTEXT_TYPE_NONE)
print("Step1",flush=True)
results = search_client.search_resources(structured_search)
for result in results.data.items:
print(result.identifier + " has availability tag checking status")
print(result.identifier + " status is " + result.lifecycle_state)
instanceId = result.identifier
resp = perform_action(signer, instanceId , 'START')
except oci.exceptions.ServiceError as e:
print('RQS Search failed with Service Error: {0}'.format(e),flush=True)
raise
except oci.exceptions.RequestException as e:
print('RQS Search failed w/ a Request exception. {0}'.format(e),flush=True)
raise
return resp
Related
I know that this error message 'TypeError: 'NoneType' object is not iterable' means that there is None is the data. However, I am looking through all of my list and there is no part which has a no value element. This is the section of my code that correlates to my issue.
url = input('\nEnter URL Here: ')
type_url = url.split('?')[0].split('/')[-1]
if type_url == 'tv-guide':
id_url = url.split('=')[-1]
else:
id_url = url.split('&')[0].split('=')[-1]
#id_url = url.split('&')[0].split('=')[-1]
#fix_id = id_url.split('=')[-1]
resp = requests.post('https://tvapi-sgn.solocoo.tv/v1/assets/{}/play'.format(id_url), headers=headers, json=json_data).json()
lic_url = resp['drm']['licenseUrl']
mpd_url = resp['url']
print(f'\nMPD URL: {mpd_url}')
print(f'\nLICENSE URL: {lic_url}')
def get_pssh_kid(keyId):
array_of_bytes = bytearray( b'\x00\x00\x002pssh\x00\x00\x00\x00')
array_of_bytes.extend(bytes.fromhex("edef8ba979d64acea3c827dcd51d21ed"))
array_of_bytes.extend(b'\x00\x00\x00\x12\x12\x10')
array_of_bytes.extend(bytes.fromhex(keyId.replace("-", "")))
return base64.b64encode(bytes.fromhex(array_of_bytes.hex()))
def get_pssh_mpd(mpd_url):
r = requests.get(url=mpd_url)
r.raise_for_status()
xml = xmltodict.parse(r.text)
mpd = json.loads(json.dumps(xml))
tracks = mpd['MPD']['Period']['AdaptationSet']
for video_tracks in tracks:
if video_tracks['#mimeType'] == 'video/mp4':
for t in video_tracks["ContentProtection"]:
if t['#schemeIdUri'].lower() == "urn:mpeg:dash:mp4protection:2011":
kid = t['#cenc:default_KID']
pssh = get_pssh_kid(kid)
return pssh
pssh = get_pssh_mpd(mpd_url).decode('utf-8')
print("\n[green]+ [INFO] PSSH: [/green]" + pssh)
def decrypt_keys():
wvdecrypt = WvDecrypt(pssh)
raw_challenge = wvdecrypt.get_challenge()
challenge_b64 = b64encode(raw_challenge).decode()
widevine_license = requests.post(
url=lic_url, data=raw_challenge, headers=headers, timeout=10)
license_b64 = b64encode(widevine_license.content)
wvdecrypt.update_license(license_b64)
keys = wvdecrypt.start_process()
if keys:
return keys
keys = decrypt_keys()
for key in keys:
print("\n[green]+ [INFO] KEY: --key [/green]" + key)
I want my code to do the same output without the error message. Do you know what part of the list could have 'None'? Thank you :)
and this is the error i get:
+ [INFO] PSSH: AAAAMnBzc2gAAAAA7e+LqXnWSs6jyCfc1R0h7QAAABISEFESrzdJJkwml6zu8TAEfX8=
unable to parse license - check protobufs
Traceback (most recent call last):
File "C:\Project\k\l3.py", line 105, in <module>
for key in keys:
TypeError: 'NoneType' object is not iterable
can someone help me fix that code? Thanks!
For some reason, wvdecrypt.start_process() returns None. As you have no default return value, that's what your function returns too.
If you want to prevent the error from breaking the execution, you could just check if keys is defined before the loop:
keys = decrypt_keys()
if keys:
for key in keys:
print("\n[green]+ [INFO] KEY: --key [/green]" + key)
However, you should probably investigate why start_process returns None first.
I am using an external library (github3.py) that defines an internal exception (github3.exceptions.UnprocessableEntity). It doesn't matter how this exception is defined, so I want to create a side effect and set the attributes I use from this exception.
Tested code not-so-minimal example:
import github3
class GithubService:
def __init__(self, token: str) -> None:
self.connection = github3.login(token=token)
self.repos = self.connection.repositories()
def create_pull(self, repo_name: str) -> str:
for repo in self.repos:
if repo.full_name == repo_name:
break
try:
created_pr = repo.create_pull(
title="title",
body="body",
head="head",
base="base",
)
except github3.exceptions.UnprocessableEntity as github_exception:
extra = ""
for error in github_exception.errors:
if "message" in error:
extra += f"{error['message']} "
else:
extra += f"Invalid field {error['field']}. " # testing this case
return f"{repo_name}: {github_exception.msg}. {extra}"
I need to set the attributes msg and also errors from the exception. So I tried in my test code using pytest-mock:
#pytest.fixture
def mock_github3_login(mocker: MockerFixture) -> MockerFixture:
"""Fixture for mocking github3.login."""
mock = mocker.patch("github3.login", autospec=True)
mock.return_value.repositories.return_value = [
mocker.Mock(full_name="staticdev/nope"),
mocker.Mock(full_name="staticdev/omg"),
]
return mock
def test_create_pull_invalid_field(
mocker: MockerFixture, mock_github3_login: MockerFixture,
) -> None:
exception_mock = mocker.Mock(errors=[{"field": "head"}], msg="Validation Failed")
mock_github3_login.return_value.repositories.return_value[1].create_pull.side_effect = github3.exceptions.UnprocessableEntity(mocker.Mock())
mock_github3_login.return_value.repositories.return_value[1].create_pull.return_value = exception_mock
response = GithubService("faketoken").create_pull("staticdev/omg")
assert response == "staticdev/omg: Validation Failed. Invalid field head."
The problem with this code is that, if you have side_effect and return_value, Python just ignores the return_value.
The problem here is that I don't want to know the implementation of UnprocessableEntity to call it passing the right arguments to it's constructor. Also, I didn't find other way using just side_effect. I also tried to using return value and setting the class of the mock and using it this way:
def test_create_pull_invalid_field(
mock_github3_login: MockerFixture,
) -> None:
exception_mock = Mock(__class__ = github3.exceptions.UnprocessableEntity, errors=[{"field": "head"}], msg="Validation Failed")
mock_github3_login.return_value.repositories.return_value[1].create_pull.return_value = exception_mock
response = GithubService("faketoken").create_pull("staticdev/omg")
assert response == "staticdev/omg: Validation Failed. Invalid field head."
This also does not work, the exception is not thrown. So I don't know how to overcome this issue given the constraint I don't want to see the implementation of UnprocessableEntity. Any ideas here?
So based on your example, you don't really need to mock github3.exceptions.UnprocessableEntity but only the incoming resp argument.
So the following test should work:
def test_create_pull_invalid_field(
mocker: MockerFixture, mock_github3_login: MockerFixture,
) -> None:
mocked_response = mocker.Mock()
mocked_response.json.return_value = {
"message": "Validation Failed", "errors": [{"field": "head"}]
}
repo = mock_github3_login.return_value.repositories.return_value[1]
repo.create_pull.side_effect = github3.exceptions.UnprocessableEntity(mocked_response)
response = GithubService("faketoken").create_pull("staticdev/omg")
assert response == "staticdev/omg: Validation Failed. Invalid field head."
EDIT:
If you want github3.exceptions.UnprocessableEntity to be completely abstracted, it won't be possible to mock the entire class as catching classes that do not inherit from BaseException is not allowed (See docs). But you can get around it by mocking the constructor only:
def test_create_pull_invalid_field(
mocker: MockerFixture, mock_github3_login: MockerFixture,
) -> None:
def _initiate_mocked_exception(self) -> None:
self.errors = [{"field": "head"}]
self.msg = "Validation Failed"
mocker.patch.object(
github3.exceptions.UnprocessableEntity, "__init__",
_initiate_mocked_exception
)
repo = mock_github3_login.return_value.repositories.return_value[1]
repo.create_pull.side_effect = github3.exceptions.UnprocessableEntity
response = GithubService("faketoken").create_pull("staticdev/omg")
assert response == "staticdev/omg: Validation Failed. Invalid field head."
This code acts as an early warning system for ADFS failures, which works fine when run locally. Problem is that when I run it in Lambda, it loops non stop.
In short:
lambda_handler() runs pagecheck()
pagecheck() produces the info needed then passes 2 lists (msgdet_list, error_list) and an int (error_count) to notification().
notification() collates and prints the output. The output is two key variables (notificationheader and notificationbody).
I've #commentedOut the SNS piece which would usually email the info, and am using print() to instead send the info to CloudWatch logs until I can get the loop sorted. Logs:
CloudWatch logs
If I run this locally, it produces a clean single output. In Lambda, the function will loop until it times out. It's almost like every time the lists are updated, they're passed to the notification() module and it's run. I can limit the function time, but would rather fix the code!
Cheers,
tac
# This python/boto3/lambda script sends a request to an Office 365 landing page, parses return details to confirm a successful redirect to /
# the organisation ADFS homepage, authenticates homepge is correct, raises any errors, and sends a consolodated report to /
# an AWS SNS topic.
# Run once to produce pageserver and htmlchar values for global variables.
# Import required modules
import boto3
import urllib.request
from urllib.request import Request, urlopen
from datetime import datetime
import time
import re
import sys
# Global variables to be set
url = "https://outlook.com/CONTOSSO.com"
adfslink = "https://sts.CONTOSSO.com/adfs/ls/?client-request-id="
# Input after first run
pageserver = "Microsoft-HTTPAPI/2.0 Microsoft-HTTPAPI/2.0"
htmlchar = 18600
# Input AWS SNS ARN
snsarn = 'arn:aws:sns:ap-southeast-2:XXXXXXXXXXXXX:Daily_Check_Notifications_CONTOSSO'
sns = boto3.client('sns')
def pagecheck():
# Present the request to the webpage as if coming from a user in a browser
user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)'
values = {'name' : 'user'}
headers = { 'User-Agent' : user_agent }
data = urllib.parse.urlencode(values)
data = data.encode('ascii')
# "Null" the Message Detail and Error lists
msgdet_list = []
error_list = []
request = Request(url)
req = urllib.request.Request(url, data, headers)
response = urlopen(request)
with urllib.request.urlopen(request) as response:
# Get the URL. This gets the real URL.
acturl = response.geturl()
msgdet_list.append("\nThe Actual URL is:")
msgdet_list.append(str(acturl))
if adfslink not in acturl:
error_list.append(str("Redirect Fail"))
# Get the HTTP resonse code
httpcode = response.code
msgdet_list.append("\nThe HTTP code is: ")
msgdet_list.append(str(httpcode))
if httpcode//200 != 1:
error_list.append(str("No HTTP 2XX Code"))
# Get the Headers as a dictionary-like object
headers = response.info()
msgdet_list.append("\nThe Headers are:")
msgdet_list.append(str(headers))
if response.info() == "":
error_list.append(str("Header Error"))
# Get the date of request and compare to UTC (DD MMM YYYY HH MM)
date = response.info()['date']
msgdet_list.append("The Date is: ")
msgdet_list.append(str(date))
returndate = str(date.split( )[1:5])
returndate = re.sub(r'[^\w\s]','',returndate)
returndate = returndate[:-2]
currentdate = datetime.utcnow()
currentdate = currentdate.strftime("%d %b %Y %H%M")
if returndate != currentdate:
date_error = ("Date Error. Returned Date: ", returndate, "Expected Date: ", currentdate, "Times in UTC (DD MMM YYYY HH MM)")
date_error = str(date_error)
date_error = re.sub(r'[^\w\s]','',date_error)
error_list.append(str(date_error))
# Get the server
headerserver = response.info()['server']
msgdet_list.append("\nThe Server is: ")
msgdet_list.append(str(headerserver))
if pageserver not in headerserver:
error_list.append(str("Server Error"))
# Get all HTML data and confirm no major change to content size by character lenth (global var: htmlchar).
html = response.read()
htmllength = len(html)
msgdet_list.append("\nHTML Length is: ")
msgdet_list.append(str(htmllength))
msgdet_list.append("\nThe Full HTML is: ")
msgdet_list.append(str(html))
msgdet_list.append("\n")
if htmllength // htmlchar != 1:
error_list.append(str("Page HTML Error - incorrect # of characters"))
if adfslink not in str(acturl):
error_list.append(str("ADFS Link Error"))
error_list.append("\n")
error_count = len(error_list)
if error_count == 1:
error_list.insert(0, 'No Errors Found.')
elif error_count == 2:
error_list.insert(0, 'Error Found:')
else:
error_list.insert(0, 'Multiple Errors Found:')
# Pass completed results and data to the notification() module
notification(msgdet_list, error_list, error_count)
# Use AWS SNS to create a notification email with the additional data generated
def notification(msgdet_list, error_list, errors):
datacheck = str("\n".join(msgdet_list))
errorcheck = str("\n".join(error_list))
notificationbody = str(errorcheck + datacheck)
if errors >1:
result = 'FAILED!'
else:
result = 'passed.'
notificationheader = ('The daily ADFS check has been marked as ' + result + ' ' + str(errors) + ' ' + str(error_list))
if result != 'passed.':
# message = sns.publish(
# TopicArn = snsarn,
# Subject = notificationheader,
# Message = notificationbody
# )
# Output result to CloudWatch logstream
print('Response: ' + notificationheader)
else:
print('passed')
sys.exit()
# Trigger the Lambda handler
def lambda_handler(event, context):
aws_account_ids = [context.invoked_function_arn.split(":")[4]]
pagecheck()
return "Successful"
sys.exit()
Your CloudWatch logs contain the following error message:
Process exited before completing request
This is caused by invoking sys.exit() in your code. Locally your Python interpreter will just terminate when encountering such a sys.exit().
AWS Lambda on the other hand expects a Python function to just return and handles sys.exit() as an error. As your function probably got invoked asynchronously AWS Lambda retries to execute it twice.
To solve your problem, you can replace the occurences of sys.exit() with return or even better, just remove the sys.exit() calls, as there would be already implicit returns in the places where you use sys.exit().
Hey I have the following function to launch a batch job.
My batch job has two parameters to be passed in
--source
--destination
def kickoff_transfer_batch(self,item):
try:
batch = boto3.client('batch')
bucket, key = get_s3_bucket_and_key(item.source)
jobName = 'transfer-'+ key
jobQueue = 'aws-strikeforce-on-demand-restore-prod'
jobDefinition = 'aws-strikeforce-transfer-prod'
source = '--source ' + item.source
destination ='--destination ' + item.destination
command = []
command.append(source)
command.append(destination)
submit_job_response = batch.submit_job(
jobName=jobName,
jobQueue=jobQueue,
jobDefinition=jobDefinition,
containerOverrides={'command': command}
)
job_id = submit_job_response['jobId']
print('Submitted job {} {} to the job queue {}'.format(jobName, job_id, jobQueue))
except Exception as err:
item.errored = True
print("failed: " + item.source)
print("error: " + str(err))
stack_trace = traceback.format_exc()
self._log_error_notes(item.source, err, stack_trace)
My job is being launched from batch, but my container is failing to launch due to how I am passing in --source and --dest.
Here is the error log
main.py: error: unrecognized arguments: --source file_test.txt --destination file_test.txt
How can I fix my command list to pass in the proper arguments.
When I launch job at the command line I would just type
--source file, --dest file
The answer to this for future reference
def kickoff_transfer_batch(self,item):
try:
batch = boto3.client('batch')
bucket, key = get_s3_bucket_and_key(item.source)
jobName = 'transfer-'+ key
jobQueue = 'aws-strikeforce-on-demand-restore-prod'
jobDefinition = 'aws-strikeforce-transfer-prod'
command = '--source '+ item.source + '--destination ' + item.destination
command = command.split()
submit_job_response = batch.submit_job(
jobName=jobName,
jobQueue=jobQueue,
jobDefinition=jobDefinition,
containerOverrides={'command': command}
)
job_id = submit_job_response['jobId']
print('Submitted job {} {} to the job queue {}'.format(jobName, job_id, jobQueue))
except Exception as err:
item.errored = True
print("failed: " + item.source)
print("error: " + str(err))
stack_trace = traceback.format_exc()
self._log_error_notes(item.source, err, stack_trace)
New to Python/Boto3, this should be an easy one but still learning :)
I have a Lambda function which creates a number of snapshots and works fine:
def create_snapshot():
volumes = ec2_client.describe_volumes(
Filters=[
{'N'...
...
for volume in volumes...
....
snap_name = 'Backup of ' + snap_desc
....
snap = ec2_client.create_snapshot(
VolumeId=vol_id,
Description=snap_desc
)
I then want to receive an email from AWS SNS to let me know which snapshots the function created, which I do using:
message = sns.publish(
TopicArn=SNSARN,
Subject=("Function executed"),
Message=("%s created" % snap_name)
)
The issue is that this creates an email for each snapshot, instead of one email listing all the snapshots. Should I create another function that calls all values produced by snap_desc, or can I send all values for snap_desc in the function? And most importantly what's the best way of doing this?
Cheers!
Scott
####################### UPDATE (Thanks #omuthu) #######################
I set an array inside and outside the loop, and put the string into the message. This produced the following being sent in one message:
The following snapshots have been created:
['vol-0e0b9a5dfb8379fc0 (Instance 1 - /dev/sda1)', 'vol-03aac6b65df64661e (Instance 4 - /dev/sda1)', 'vol-0fdde765dfg452631 (Instance 2 - /dev/sda1)', 'vol-0693a9568b11f625f (Instance 3 - /dev/sda1)', etc.
Okay got it sorted, finally!
def create_snapshot():
volumes = ec2_client.describe_volumes(
Filters=[
{'N'...
...
inst_list = []
for volume in volumes...
vol_id = volume['VolumeId']
....
snap_desc = vol_id
for name in volume['Tags']:
tag_key = name['Key']
tag_val = name['Value']
if tag_key == 'Name':
snap_desc = vol_id + ' (' + tag_val + ')'
....
....
....
if backup_mod is False or (current_hour + 10) % backup_mod != 0:
...
continue
else:
print("%s is scheduled this hour" % vol_id)
for name in volume['Tags']:
inst_tag_key = name['Key']
inst_tag_val = name['Value']
if inst_tag_key == 'Name':
inst_list.append(inst_tag_val)
snap = ec2_client.create_snapshot(
VolumeId=vol_id,
Description=snap_desc,
)
print("%s created" % snap['SnapshotId'])
msg = str("\n".join(inst_list))
if len(inst_list) != 0:
message = sns.publish(
TopicArn=SNSARN,
Subject=("Daily Lambda snapshot function complete"),
Message=("The following snapshots have been created today:\n\n" + msg + "\n")
)
print("Response: {}".format(message))