How do I catch google.auth.exceptions.RefreshError in python subscriber? - python-3.x

Question

I am trying to catch the following exception but am not having any luck.
ERROR 2019-03-04 16:31:50,522 _plugin_wrapping 15 140108141426432 AuthMetadataPluginCallback "<google.auth.transport.grpc.AuthMetadataPlugin object at 0x7f6dd8330b70>" raised exception!
Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/grpc/_plugin_wrapping.py", line 79, in __call__
callback_state, callback))
File "/usr/local/lib/python3.6/site-packages/google/auth/transport/grpc.py", line 77, in __call__
callback(self._get_authorization_headers(context), None)
File "/usr/local/lib/python3.6/site-packages/google/auth/transport/grpc.py", line 65, in _get_authorization_headers
headers)
File "/usr/local/lib/python3.6/site-packages/google/auth/credentials.py", line 122, in before_request
self.refresh(request)
File "/usr/local/lib/python3.6/site-packages/google/oauth2/service_account.py", line 322, in refresh
request, self._token_uri, assertion)
File "/usr/local/lib/python3.6/site-packages/google/oauth2/_client.py", line 145, in jwt_grant
response_data = _token_endpoint_request(request, token_uri, body)
File "/usr/local/lib/python3.6/site-packages/google/oauth2/_client.py", line 111, in _token_endpoint_request
_handle_error_response(response_body)
File "/usr/local/lib/python3.6/site-packages/google/oauth2/_client.py", line 61, in _handle_error_response
error_details, response_body)
google.auth.exceptions.RefreshError: ('invalid_grant: Invalid JWT Signature.', '{\n "error": "invalid_grant",\n "error_description": "Invalid JWT Signature."\n}')
I am using google cloud pubsub_v1 and this portion is the subscriber.
I want to catch this exception so the client can request the credentials again. Requesting credentials API's we made are working fine. To get this error I manually deleted the key from the SA using google cloud console UI and commented out the startup function call to get credentials. So it is trying to use the old ones.
In the code I think I have what is a pretty basic subscriber.
from google.cloud import pubsub_v1
from google.oauth2 import service_account
from google.api_core.exceptions import NotFound
from google.auth.exceptions import RefreshError
...
def startSubscribe(self):
project, subscription, gauth = self.decryptCreds()
credentials = service_account.Credentials.from_service_account_info(gauth)
subscriber = pubsub_v1.SubscriberClient(credentials=credentials)
subscription_path = subscriber.subscription_path(project, subscription)
self.future = subscriber.subscribe(subscription_path, callback=self.callback)
LOGGER.info('Listening for messages on {}'.format(subscription_path))
while True:
try:
LOGGER.info("Calling future.result()")
self.future.result()
self.future.exception()
# TODO: Need to figure out permission error, etc... and handle them properly. This is just a catch all until
# I figure out more about gcp subscribe and futures
except RefreshError as e:
# Catch permission exception and get creds again
LOGGER.info("Caught the correct error")
LOGGER.error(e, exc_info=True)
except Exception as e:
LOGGER.info("Must be something else")
LOGGER.error(e, exc_info=True)
LOGGER.info("Looping")
The startSubscribe function is also wrapped in a try/except but I do not get the error there either.
Does anyone have any experience or know how to catch this exception or have any insight how to check credentials so I can request new ones?
Thank you.

Answer 1

We contacted google and they reported back that there is a bug in the async (using the futures) version.
So I modified our code to do it synchronously.
subscriber = None
while self.alive:
try:
if subscriber is None:
project, subscription, gauth = self.decryptCreds()
credentials = (service_account.Credentials
.from_service_account_info(gauth))
subscriber = (pubsub_v1
.SubscriberClient(credentials=credentials))
subscription_path = (subscriber
.subscription_path(project,
subscription))
pull_response = subscriber.pull(subscription_path,
max_messages=NUM_MESSAGES,
timeout=60,
retry=None)
for msg in pull_response.received_messages:
msg_id = msg.message.message_id
try:
payload = json.loads(
base64.b64decode(msg.message.data.decode('utf-8'))
.decode('utf-8'))
LOGGER.info("Got Message")
LOGGER.debug(payload)
LOGGER.debug(type(payload))
# Removed for privacy...
except Exception as e:
LOGGER.info(e, exc_info=True)
# Here we are ack-ing the message no matter what
# because if the message itself is bad, there is no way
# to tell the publisher and we do not want bad messages
# continually sent.
subscriber.acknowledge(subscription_path, [msg.ack_id])
except NotFound as nf:
LOGGER.error(nf, exc_info=True)
try:
# If we get here try to get the credentials before remaking
# the subscriber
self.cpps.doGetCreds()
# Close the channel first to make sure not file handlers
# left open
try:
subscriber.api.transport._channel.close()
except Exception as e:
LOGGER.error(e, exc_info=True)
subscriber = None
except Exception as e:
LOGGER.error(
'Exception here may have been crashing firmware.')
LOGGER.error(e, exc_info=True)
except (RefreshError, ServiceUnavailable) as re:
LOGGER.error(re, exc_info=True)
except RetryError as e:
LOGGER.error(e, exc_info=True)
except Exception as e:
LOGGER.error(e, exc_info=True)
time.sleep(10)
LOGGER.info("Exiting Cloud Pull")
try:
subscriber.api.transport._channel.close()
except Exception as e:
LOGGER.error(e, exc_info=True)
The important thing to note here, as it is not described in the documentation is subscriber.api.transport._channel.close(). Otherwise it will leave lots of file handles open.
So basically just switch to synchronous pulling and make sure to close the channel.
I hope that helps #ProGirlXOXO
Oh and the relevant google imports are
from google.cloud import pubsub_v1
from google.oauth2 import service_account
from google.auth.exceptions import RefreshError
from google.api_core.exceptions import RetryError, ServiceUnavailable, NotFound