I am getting an error when using python GEE api to save processed image in Colab. The code and the error are as follow:
# Load a landsat image and select three bands.
landsat = ee.Image('LANDSAT/LC08/C01/T1_TOA/LC08_123032_20140515')\
.select(['B4', 'B3', 'B2'])
# Create a geometry representing an export region.
geometry = ee.Geometry.Rectangle([116.2621, 39.8412, 116.4849, 40.01236])
center = geometry.centroid().getInfo()['coordinates']
center.reverse()
Mapdisplay(center,{'landsat':landsat.getMapId()},zoom_start=7)
# Get band 4 from the Landsat image, copy it.
band4 = landsat.select('B4').rename('b4_mean')\
.addBands(landsat.select('B4').rename('b4_sample'))\
.addBands(landsat.select('B4').rename('b4_max'))\
# Export the image to an Earth Engine asset.
task = ee.batch.Export.image.toAsset(**{
'image': band4,
'description': 'imageToAssetExample',
'assetId': 'users/csaybar/exampleExport',
'scale': 100,
'region': geometry.getInfo()['coordinates']
})
task.start()
This error occurs due to "task.start()"
WARNING:googleapiclient.http:Invalid JSON content from response: b'{\n "error": {\n "code": 403,\n "message": "Request had insufficient authentication scopes.",\n "status": "PERMISSION_DENIED",\n "details": [\n {\n "#type": "type.googleapis.com/google.rpc.ErrorInfo",\n "reason": "ACCESS_TOKEN_SCOPE_INSUFFICIENT",\n "domain": "googleapis.com",\n "metadata": {\n "method": "google.earthengine.v1alpha.EarthEngine.ExportImage",\n "service": "earthengine.googleapis.com"\n }\n }\n ]\n }\n}\n'
---------------------------------------------------------------------------
HttpError Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/ee/data.py in _execute_cloud_call(call, num_retries)
327 try:
--> 328 return call.execute(num_retries=num_retries)
329 except googleapiclient.errors.HttpError as e:
6 frames
HttpError: <HttpError 403 when requesting https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/image:export?alt=json returned "Request had insufficient authentication scopes.". Details: "[{'#type': 'type.googleapis.com/google.rpc.ErrorInfo', 'reason': 'ACCESS_TOKEN_SCOPE_INSUFFICIENT', 'domain': 'googleapis.com', 'metadata': {'method': 'google.earthengine.v1alpha.EarthEngine.ExportImage', 'service': 'earthengine.googleapis.com'}}]">
During handling of the above exception, another exception occurred:
EEException Traceback (most recent call last)
/usr/local/lib/python3.8/dist-packages/ee/data.py in _execute_cloud_call(call, num_retries)
328 return call.execute(num_retries=num_retries)
329 except googleapiclient.errors.HttpError as e:
--> 330 raise _translate_cloud_exception(e)
331
332
EEException: Request had insufficient authentication scopes.
I tried to look for a GEE service in the google cloud console to enable it, but couldn't find one. And I want to konw how to fix this problem.
The most recent version of the google earth engine API seems to have introduced some issues with gcloud. Try rolling back to an earlier version using conda install -c conda-forge earthengine-api=0.1.320 and then rerunning your process.
For future reference, most GEE-related questions get asked and answered over at gis.stackexchange.com.
Related
I am trying to write a small program using the AzureML Python SDK (v1.0.85) to register an Environment in AMLS and use that definition to construct a local Conda environment when experiments are being run (for a pre-trained model). The code works fine for simple scenarios where all dependencies are loaded from Conda/ public PyPI, but when I introduce a private dependency (e.g. a utils library) I am getting a InternalServerError with the message "Error getting recipe specifications".
The code I am using to register the environment is (after having authenticated to Azure and connected to our workspace):
environment_name = config['environment']['name']
py_version = "3.7"
conda_packages = ["pip"]
pip_packages = ["azureml-defaults"]
private_packages = ["./env-wheels/utils-0.0.3-py3-none-any.whl"]
print(f"Creating environment with name {environment_name}")
environment = Environment(name=environment_name)
conda_deps = CondaDependencies()
print(f"Adding Python version: {py_version}")
conda_deps.set_python_version(py_version)
for conda_pkg in conda_packages:
print(f"Adding Conda denpendency: {conda_pkg}")
conda_deps.add_conda_package(conda_pkg)
for pip_pkg in pip_packages:
print(f"Adding Pip dependency: {pip_pkg}")
conda_deps.add_pip_package(pip_pkg)
for private_pkg in private_packages:
print(f"Uploading private wheel from {private_pkg}")
private_pkg_url = Environment.add_private_pip_wheel(workspace=ws, file_path=Path(private_pkg).absolute(), exist_ok=True)
print(f"Adding private Pip dependency: {private_pkg_url}")
conda_deps.add_pip_package(private_pkg_url)
environment.python.conda_dependencies = conda_deps
environment.register(workspace=ws)
And the code I am using to create the local Conda environment is:
amls_environment = Environment.get(ws, name=environment_name, version=environment_version)
print(f"Building environment...")
amls_environment.build_local(workspace=ws)
The exact error message being returned when build_local(...) is called is:
Traceback (most recent call last):
File "C:\Anaconda\envs\AMLSExperiment\lib\site-packages\azureml\core\environment.py", line 814, in build_local
raise error
File "C:\Anaconda\envs\AMLSExperiment\lib\site-packages\azureml\core\environment.py", line 807, in build_local
recipe = environment_client._get_recipe_for_build(name=self.name, version=self.version, **payload)
File "C:\Anaconda\envs\AMLSExperiment\lib\site-packages\azureml\_restclient\environment_client.py", line 171, in _get_recipe_for_build
raise Exception(message)
Exception: Error getting recipe specifications. Code: 500
: {
"error": {
"code": "ServiceError",
"message": "InternalServerError",
"detailsUri": null,
"target": null,
"details": [],
"innerError": null,
"debugInfo": null
},
"correlation": {
"operation": "15043e1469e85a4c96a3c18c45a2af67",
"request": "19231be75a2b8192"
},
"environment": "westeurope",
"location": "westeurope",
"time": "2020-02-28T09:38:47.8900715+00:00"
}
Process finished with exit code 1
Has anyone seen this error before or able to provide some guidance around what the issue may be?
The issue was with out firewall blocking the required requests between AMLS and the storage container (I presume to get the environment definitions/ private wheels).
We resolved this by updating the firewall with appropriate ALLOW rules for the AMLS service to contact and read from the attached storage container.
Assuming that you'd like to run in the script on a remote compute, then my suggestion would be to pass the environment you just "got". to a RunConfiguration, then pass that to an ScriptRunConfig, Estimator, or a PythonScriptStep
from azureml.core import ScriptRunConfig
from azureml.core.runconfig import DEFAULT_CPU_IMAGE
src = ScriptRunConfig(source_directory=project_folder, script='train.py')
# Set compute target to the one created in previous step
src.run_config.target = cpu_cluster.name
# Set environment
amls_environment = Environment.get(ws, name=environment_name, version=environment_version)
src.run_config.environment = amls_environment
run = experiment.submit(config=src)
run
Check out the rest of the notebook here.
If you're looking for a local run this notebook might help.
I have the following problem:
my application is a bridge between dbus and json-rpc.
I have a dbus library that in case of Dbus exception is raising a
DBusException("my err msg") exception.
My json-rpc library is catching all the Exception and convert it to a nice generic server error message
{"jsonrpc": "2.0", "error": {"code": -32000, "message": "Server error"}, "id": 2}
The problem is that i want to handle better those exceptions and I can distinguish between them only using the error message:
for example I need to convert the
DBusException("Invalid Parameters")
to
{"jsonrpc": "2.0", "error": {"code": -32001, "message": "Server error", data="Invalid Parameters" }, "id": 2}
that can be done raising in my library an ApiError(msg, code, data) exception.
So summarizing:
- I need to catch a DBusException
- based on the err_msg I need to switch between different exception types and re-raise a modified
ApiError exception
How is the better approach to that? this needs to be done in at least 10 functions that have the same exceptions.
1) using a decorator?
def my_exception_catcher(fun, *args, **kwargv):
try:
fun(args, *argv)
except DBusException as e
err_msg = str(e)
if err_msg == "Invalid Arguments":
raise ApiError("Server Error", code=1, data=err_msg)
else if err_msg == "Connect Error":
raise ApiError("Server Error", code=2, data=err_msg)
else:
raise
#my_exception_catcher
my_fun(*args):
do_something
2) using a function to determine the exception type?
def find_my_dbus_error(err_msg):
if err_msg == "Invalid Arguments":
return ApiError("Server Error", code=1, data=err_msg)
else if err_msg == "Connect Error":
return ApiError("Server Error", code=2, data=err_msg)
else:
return Exception(err_msg)
try:
my_fun(params)
except DBusException as e
raise find_my_dbus_error(err_msg)
3) something else?
Thanks
Nick
I'm trying to create a connection using a boto3 python script. It's a connection to a mysql database on an ec2 instance. I'm using the script below. I'm getting the error message below. I'm not sure what I'm missing. I've used similar scripts to create glue crawlers without issue. I'm pretty much following the boto3 documentation except I added username and password in the ConnectionProperties. I'm not sure if that's correct. Can someone please let me know if I'm doing this right, or else what I need to fix in my code?
code:
# create new connection
response = client.create_connection(
ConnectionInput={
'Name': 'tst_scrpt',
'ConnectionType': 'JDBC',
'ConnectionProperties': {
'string': 'jdbc:mysql://dataxxx:3306/disxxx',
'username':'root',
'password':'ip1k5PNCxxxxx'
}
,
'PhysicalConnectionRequirements': {
'SubnetId': 'subnet-0436167b7cbxxxx',
'SecurityGroupIdList': [
'sg-02c3f45ce51exxxxx'
]
}
}
)
Error:
---------------------------------------------------------------------------
InvalidInputException Traceback (most recent call last)
<ipython-input-18-6ac0bdcfa816> in <module>
16 'SubnetId': 'subnet-0436167b7cbxxxx',
17 'SecurityGroupIdList': [
---> 18 'sg-02c3f45ce51exxxxx'
19 ]
20 # ,
/anaconda3/envs/py36/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
355 "%s() only accepts keyword arguments." % py_operation_name)
356 # The "self" in this scope is referring to the BaseClient.
--> 357 return self._make_api_call(operation_name, kwargs)
358
359 _api_call.__name__ = str(py_operation_name)
/anaconda3/envs/py36/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
659 error_code = parsed_response.get("Error", {}).get("Code")
660 error_class = self.exceptions.from_code(error_code)
--> 661 raise error_class(parsed_response, operation_name)
662 else:
663 return parsed_response
InvalidInputException: An error occurred (InvalidInputException) when calling the CreateConnection operation: Validation for connection properties failed
I had the ConnectionProperties wrong. replace 'string' with 'JDBC_CONNECTION_URL'. The correction is below.
'ConnectionProperties': {
'JDBC_CONNECTION_URL': 'jdbc:mysql://dataxxx:3306/disxxx',
'username':'root',
'password':'ip1k5PNCxxxxx'
}
Sorry for the long post. I am trying to call a Lex bot with the PostText runtime API with my lambda function. However when I test this call then it returns that the userID is not authorized to use this. This is the error message I receive:
Response:
{
"errorMessage": "An error occurred (AccessDeniedException) when calling the PostText operation: User: arn:aws:sts::981709171824:assumed-role/lambda_basic_execution/OrchestratorAPIApp is not authorized to perform: lex:PostText on resource: arn:aws:lex:us-east-1:981709171824:bot:SupportBot_BookCab:SupportBot_BookCab",
"errorType": "ClientError",
"stackTrace": [
[
"/var/task/lambda_function.py",
18,
"lambda_handler",
"inputText= userInput"
],
[
"/var/runtime/botocore/client.py",
314,
"_api_call",
"return self._make_api_call(operation_name, kwargs)"
],
[
"/var/runtime/botocore/client.py",
612,
"_make_api_call",
"raise error_class(parsed_response, operation_name)"
]
]
}
Request ID:
"677f1820-6ed2-11e8-b891-33ab1951c65f"
Function Logs:
START RequestId: 677f1820-6ed2-11e8-b891-33ab1951c65f Version: $LATEST
An error occurred (AccessDeniedException) when calling the PostText operation: User: arn:aws:sts::981709171824:assumed-role/lambda_basic_execution/OrchestratorAPIApp is not authorized to perform: lex:PostText on resource: arn:aws:lex:us-east-1:981709171824:bot:SupportBot_BookCab:SupportBot_BookCab: ClientError
Traceback (most recent call last):
File "/var/task/lambda_function.py", line 18, in lambda_handler
inputText= userInput
File "/var/runtime/botocore/client.py", line 314, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/var/runtime/botocore/client.py", line 612, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (AccessDeniedException) when calling the PostText operation: User: arn:aws:sts::981709171824:assumed-role/lambda_basic_execution/OrchestratorAPIApp is not authorized to perform: lex:PostText on resource: arn:aws:lex:us-east-1:981709171824:bot:SupportBot_BookCab:SupportBot_BookCab
END RequestId: 677f1820-6ed2-11e8-b891-33ab1951c65f
REPORT RequestId: 677f1820-6ed2-11e8-b891-33ab1951c65f Duration: 325.25 ms Billed Duration: 400 ms Memory Size: 128 MB Max Memory Used: 31 MB
This is my code to calling the API:
import boto3
def lambda_handler(event, context):
responderName = event["DestinationBot"]
userId = event["RecipientID"]
userInput = event["message"]["text"]
client = boto3.client('lex-runtime')
response = client.post_text(
botName=responderName,
botAlias=responderName,
userId=userId,
sessionAttributes={
},
requestAttributes={
},
inputText= userInput
)
This is my sample test input:
{
"DestinationBot": "SupportBot_BookCab",
"RecipientID": "12345",
"message": {
"text": "book me a cab"
}
}
The userID of PostText is the way you persist the conversation back and forth between the user and Lex. It can be anything that you can identify the user by in their incoming request that is consistent and unique to them, at least for that session.
From AWS PostText Docs:
userID
The ID of the client application user. Amazon Lex uses this to identify a user's conversation with your bot. At runtime, each request must contain the userID field.
...
Length Constraints: Minimum length of 2. Maximum length of 100.
Pattern: [0-9a-zA-Z._:-]+
So if a user is using Facebook messenger, they will have a Facebook ID that is passed with their messages and you can use that as their userID.
If a user is using Twilio-SMS, they will have a phone number passed with their messages and you can use that as their userID.
Your code is currently taking event["RecipientID"] and using that as a userID. But the RecipientID from an incoming message is yourself, the receiver of the incoming message.
Your error is telling you that
... User: arn:aws:sts::XXXXXXXXXX:assumed-role/lambda_basic_execution/OrchestratorAPIApp
So the userID = event["RecipientID"] = 'arn:aws:sts::XXXXXXXX:assumed-role/lambda_basic_execution/OrchestratorAPIApp'
You definitely don't want the Recipient ID to be used.
Instead you want the sender's ID to be the userID. Something like:
userId = event["SenderID"]
That might not be the exact code you use, its just an example. You should be able to view the incoming request, and locate something in there to use as a proper userID as I explained above with Facebook and Twilio.
I've downloaded a code sample that looks for particular keywords and pulls some metrics. I've noticed a lot of Google Adwords API examples are compliant with python 3.x so I'm wondering if there is an issue with that? See below for code sample:
from googleads import adwords
def main(client):
# Initialize appropriate service.
traffic_estimator_service = client.GetService(
'TrafficEstimatorService', version='v201609')
# Construct selector object and retrieve traffic estimates.
keywords = [
{'text': 'mars cruise', 'matchType': 'BROAD'},
{'text': 'cheap cruise', 'matchType': 'PHRASE'},
{'text': 'cruise', 'matchType': 'EXACT'}
]
negative_keywords = [
{'text': 'moon walk', 'matchType': 'BROAD'}
]
keyword_estimate_requests = []
for keyword in keywords:
keyword_estimate_requests.append({
'keyword': {
'xsi_type': 'Keyword',
'matchType': keyword['matchType'],
'text': keyword['text']
}
})
for keyword in negative_keywords:
keyword_estimate_requests.append({
'keyword': {
'xsi_type': 'Keyword',
'matchType': keyword['matchType'],
'text': keyword['text']
},
'isNegative': 'true'
})
# Create ad group estimate requests.
adgroup_estimate_requests = [{
'keywordEstimateRequests': keyword_estimate_requests,
'maxCpc': {
'xsi_type': 'Money',
'microAmount': '1000000'
}
}]
# Create campaign estimate requests.
campaign_estimate_requests = [{
'adGroupEstimateRequests': adgroup_estimate_requests,
'criteria': [
{
'xsi_type': 'Location',
'id': '2840' # United States.
},
{
'xsi_type': 'Language',
'id': '1000' # English.
}
],
}]
# Create the selector.
selector = {
'campaignEstimateRequests': campaign_estimate_requests,
}
# Optional: Request a list of campaign-level estimates segmented by
# platform.
selector['platformEstimateRequested'] = True
# Get traffic estimates.
estimates = traffic_estimator_service.get(selector)
campaign_estimate = estimates['campaignEstimates'][0]
# Display the campaign level estimates segmented by platform.
if 'platformEstimates' in campaign_estimate:
platform_template = ('Results for the platform with ID: "%d" and name: '
'"%s".')
for platform_estimate in campaign_estimate['platformEstimates']:
platform = platform_estimate['platform']
DisplayEstimate(platform_template % (platform['id'],
platform['platformName']),
platform_estimate['minEstimate'],
platform_estimate['maxEstimate'])
# Display the keyword estimates.
if 'adGroupEstimates' in campaign_estimate:
ad_group_estimate = campaign_estimate['adGroupEstimates'][0]
if 'keywordEstimates' in ad_group_estimate:
keyword_estimates = ad_group_estimate['keywordEstimates']
keyword_template = ('Results for the keyword with text "%s" and match '
'type "%s":')
keyword_estimates_and_requests = zip(keyword_estimates,
keyword_estimate_requests)
for keyword_tuple in keyword_estimates_and_requests:
if keyword_tuple[1].get('isNegative', False):
continue
keyword = keyword_tuple[1]['keyword']
keyword_estimate = keyword_tuple[0]
DisplayEstimate(keyword_template % (keyword['text'],
keyword['matchType']),
keyword_estimate['min'], keyword_estimate['max'])
def _CalculateMean(min_est, max_est):
if min_est and max_est:
return (float(min_est) + float(max_est)) / 2.0
else:
return None
def _FormatMean(mean):
if mean:
return '%.2f' % mean
else:
return 'N/A'
def DisplayEstimate(message, min_estimate, max_estimate):
"""Displays mean average cpc, position, clicks, and total cost for estimate.
Args:
message: str message to display for the given estimate.
min_estimate: sudsobject containing a minimum estimate from the
TrafficEstimatorService response.
max_estimate: sudsobject containing a maximum estimate from the
TrafficEstimatorService response.
"""
# Find the mean of the min and max values.
mean_avg_cpc = (_CalculateMean(min_estimate['averageCpc']['microAmount'],
max_estimate['averageCpc']['microAmount'])
if 'averageCpc' in min_estimate else None)
mean_avg_pos = (_CalculateMean(min_estimate['averagePosition'],
max_estimate['averagePosition'])
if 'averagePosition' in min_estimate else None)
mean_clicks = _CalculateMean(min_estimate['clicksPerDay'],
max_estimate['clicksPerDay'])
mean_total_cost = _CalculateMean(min_estimate['totalCost']['microAmount'],
max_estimate['totalCost']['microAmount'])
print (message)
print ('Estimated average CPC: %s' % _FormatMean(mean_avg_cpc))
print ('Estimated ad position: %s' % _FormatMean(mean_avg_pos))
print ('Estimated daily clicks: %s' % _FormatMean(mean_clicks))
print ('Estimated daily cost: %s' % _FormatMean(mean_total_cost))
if __name__ == '__main__':
# Initialize client object.
adwords_client = adwords.AdWordsClient.LoadFromStorage()
main(adwords_client)
Here is the error message:
(Money) not-found
path: "Money", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Keyword) not-found
path: "Keyword", not-found
(Location) not-found
path: "Location", not-found
(Language) not-found
path: "Language", not-found
<suds.sax.document.Document object at 0x03BF1D10>
Server raised fault in response.
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 82, in send
fp = self.u2open(u2request)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 132, in u2open
return url.open(u2request, timeout=tm)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 472, in open
response = meth(req, response)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 582, in http_response
'http', request, response, code, msg, hdrs)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 510, in error
return self._call_chain(*args)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 444, in _call_chain
result = func(*args)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\urllib\request.py", line 590, in http_error_default
raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 500: Internal Server Error
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 613, in send
reply = self.options.transport.send(request)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\transport\http.py", line 94, in send
raise TransportError(e.msg, e.code, e.fp)
suds.transport.TransportError: Internal Server Error
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\adwords test - Copy (2).py", line 177, in <module>
main(adwords_client)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\adwords test - Copy (2).py", line 95, in main
estimates = traffic_estimator_service.get(selector)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\googleads\common.py", line 696, in MakeSoapRequest
raise e
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\googleads\common.py", line 692, in MakeSoapRequest
for arg in args])
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 521, in __call__
return client.invoke(args, kwargs)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 581, in invoke
result = self.send(soapenv)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 619, in send
description=tostr(e), original_soapenv=original_soapenv)
File "C:\Users\sfroese\AppData\Local\Programs\Python\Python35-32\lib\site-packages\suds\client.py", line 670, in process_reply
raise WebFault(fault, replyroot)
suds.WebFault: Server raised fault: '[AuthenticationError.CLIENT_CUSTOMER_ID_IS_REQUIRED # ; trigger:'<null>']'
You should set client_customer_id in your googleads.yaml file . you can get client_customer_id from your manager account. go to your manager account and add client then copy your id from upright corner of screen. in googleads.yaml file paste that id to client_customer_id .