fix: handle disabled-region timeouts and Python 3.12 import compatibility

principalmapper/graphing/autoscaling_edges.py

@@ -19,7 +19,8 @@
 import logging
 from typing import Dict, List, Optional
 
-from botocore.exceptions import ClientError
+from botocore.config import Config
+from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError
 
 from principalmapper.common import Edge, Node
 from principalmapper.graphing.edge_checker import EdgeChecker
@@ -50,7 +51,11 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]]
         if self.session is not None:
             as_regions = botocore_tools.get_regions_to_search(self.session, 'autoscaling', region_allow_list, region_deny_list)
             for region in as_regions:
-                autoscaling_clients.append(self.session.create_client('autoscaling', region_name=region, **asargs))
+                autoscaling_clients.append(self.session.create_client(
+                    'autoscaling', region_name=region,
+                    config=Config(connect_timeout=10, retries={'max_attempts': 0}),
+                    **asargs
+                ))
 
         launch_configs = []
         for as_client in autoscaling_clients:
@@ -66,7 +71,7 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]]
                                     'lc_iip': launch_config['IamInstanceProfile']
                                 })
 
-            except ClientError as ex:
+            except (ClientError, ConnectTimeoutError, EndpointConnectionError) as ex:
                 logger.warning('Unable to search region {} for launch configs. The region may be disabled, or the error may '
                                'be caused by an authorization issue. Continuing.'.format(as_client.meta.region_name))
                 logger.debug('Exception details: {}'.format(ex))

principalmapper/graphing/cloudformation_edges.py

@@ -20,7 +20,8 @@
 import os
 from typing import List, Optional
 
-from botocore.exceptions import ClientError
+from botocore.config import Config
+from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError
 
 from principalmapper.common import Edge, Node
 from principalmapper.graphing.edge_checker import EdgeChecker
@@ -52,7 +53,11 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]]
         if self.session is not None:
             cf_regions = botocore_tools.get_regions_to_search(self.session, 'cloudformation', region_allow_list, region_deny_list)
             for region in cf_regions:
-                cloudformation_clients.append(self.session.create_client('cloudformation', region_name=region, **cfargs))
+                cloudformation_clients.append(self.session.create_client(
+                    'cloudformation', region_name=region,
+                    config=Config(connect_timeout=10, retries={'max_attempts': 0}),
+                    **cfargs
+                ))
 
         # grab existing cloudformation stacks
         stack_list = []
@@ -65,7 +70,7 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]]
                         if stack['StackStatus'] not in ['CREATE_FAILED', 'DELETE_COMPLETE', 'DELETE_FAILED',
                                                         'DELETE_IN_PROGRESS']:  # ignore unusable stacks
                             stack_list.append(stack)
-            except ClientError as ex:
+            except (ClientError, ConnectTimeoutError, EndpointConnectionError) as ex:
                 logger.warning('Unable to search region {} for stacks. The region may be disabled, or the error may '
                                'be caused by an authorization issue. Continuing.'.format(cf_client.meta.region_name))
                 logger.debug('Exception details: {}'.format(ex))

principalmapper/graphing/codebuild_edges.py

@@ -19,7 +19,8 @@
 import logging
 from typing import Dict, List, Optional
 
-from botocore.exceptions import ClientError
+from botocore.config import Config
+from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError
 
 from principalmapper.common import Edge, Node
 from principalmapper.graphing.edge_checker import EdgeChecker
@@ -51,7 +52,11 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]]
         if self.session is not None:
             cf_regions = botocore_tools.get_regions_to_search(self.session, 'codebuild', region_allow_list, region_deny_list)
             for region in cf_regions:
-                codebuild_clients.append(self.session.create_client('codebuild', region_name=region, **cbargs))
+                codebuild_clients.append(self.session.create_client(
+                    'codebuild', region_name=region,
+                    config=Config(connect_timeout=10, retries={'max_attempts': 0}),
+                    **cbargs
+                ))
 
         codebuild_projects = []
         for cb_client in codebuild_clients:
@@ -75,7 +80,7 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]]
                                     'project_tags': project_data['tags']
                                 })
 
-            except ClientError as ex:
+            except (ClientError, ConnectTimeoutError, EndpointConnectionError) as ex:
                 logger.warning('Unable to search region {} for projects. The region may be disabled, or the error may '
                                'be caused by an authorization issue. Continuing.'.format(cb_client.meta.region_name))
                 logger.debug('Exception details: {}'.format(ex))

principalmapper/graphing/gathering.py

@@ -22,6 +22,8 @@
 
 import botocore.session
 import botocore.exceptions
+from botocore.config import Config
+from botocore.exceptions import ConnectTimeoutError, EndpointConnectionError
 import principalmapper
 from principalmapper.common import Node, Group, Policy, Graph, OrganizationTree, OrganizationNode, OrganizationAccount
 from principalmapper.graphing import edge_identification
@@ -337,7 +339,9 @@ def get_kms_key_policies(session: botocore.session.Session, region_allow_list: O
         try:
             # Grab the keys
             cmks = []
-            kmsclient = session.create_client('kms', region_name=kms_region, **kmsargs)
+            kmsclient = session.create_client('kms', region_name=kms_region,
+                                              config=Config(connect_timeout=10, retries={'max_attempts': 0}),
+                                              **kmsargs)
             kms_paginator = kmsclient.get_paginator('list_keys')
             for page in kms_paginator.paginate():
                 cmks.extend([x['KeyArn'] for x in page['Keys']])
@@ -351,7 +355,7 @@ def get_kms_key_policies(session: botocore.session.Session, region_allow_list: O
                     json.loads(policy_str)
                 ))
                 logger.info('Caching policy for {}'.format(cmk))
-        except botocore.exceptions.ClientError as ex:
+        except (botocore.exceptions.ClientError, ConnectTimeoutError, EndpointConnectionError) as ex:
             logger.info('Unable to search KMS in region {} for key policies. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'.format(kms_region))
             logger.debug('Exception was: {}'.format(ex))
             continue
@@ -376,7 +380,9 @@ def get_sns_topic_policies(session: botocore.session.Session, region_allow_list:
         try:
             # Grab the topics
             topics = []
-            snsclient = session.create_client('sns', region_name=sns_region, **snsargs)
+            snsclient = session.create_client('sns', region_name=sns_region,
+                                              config=Config(connect_timeout=10, retries={'max_attempts': 0}),
+                                              **snsargs)
             sns_paginator = snsclient.get_paginator('list_topics')
             for page in sns_paginator.paginate():
                 topics.extend([x['TopicArn'] for x in page['Topics']])
@@ -390,7 +396,7 @@ def get_sns_topic_policies(session: botocore.session.Session, region_allow_list:
                     json.loads(policy_str)
                 ))
                 logger.info('Caching policy for {}'.format(topic))
-        except botocore.exceptions.ClientError as ex:
+        except (botocore.exceptions.ClientError, ConnectTimeoutError, EndpointConnectionError) as ex:
             logger.info('Unable to search SNS in region {} for topic policies. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'.format(sns_region))
             logger.debug('Exception was: {}'.format(ex))
             continue
@@ -416,7 +422,9 @@ def get_sqs_queue_policies(session: botocore.session.Session, account_id: str,
         try:
             # Grab the queue names
             queue_urls = []
-            sqsclient = session.create_client('sqs', region_name=sqs_region, **sqsargs)
+            sqsclient = session.create_client('sqs', region_name=sqs_region,
+                                              config=Config(connect_timeout=10, retries={'max_attempts': 0}),
+                                              **sqsargs)
             response = sqsclient.list_queues()
             if 'QueueUrls' in response:
                 queue_urls.extend(response['QueueUrls'])
@@ -445,7 +453,7 @@ def get_sqs_queue_policies(session: botocore.session.Session, account_id: str,
                         }
                     ))
                     logger.info('Queue {} does not have a queue policy, adding a "stub" policy instead.'.format(queue_name))
-        except botocore.exceptions.ClientError as ex:
+        except (botocore.exceptions.ClientError, ConnectTimeoutError, EndpointConnectionError) as ex:
             logger.info('Unable to search SQS in region {} for queues. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'.format(sqs_region))
             logger.debug('Exception was: {}'.format(ex))
 
@@ -469,7 +477,9 @@ def get_secrets_manager_policies(session: botocore.session.Session, region_allow
         try:
             # Grab the ARNs of the secrets in this region
             secret_arns = []
-            smclient = session.create_client('secretsmanager', region_name=sm_region, **smargs)
+            smclient = session.create_client('secretsmanager', region_name=sm_region,
+                                             config=Config(connect_timeout=10, retries={'max_attempts': 0}),
+                                             **smargs)
             list_secrets_paginator = smclient.get_paginator('list_secrets')
             for page in list_secrets_paginator.paginate():
                 if 'SecretList' in page:
@@ -502,7 +512,7 @@ def get_secrets_manager_policies(session: botocore.session.Session, region_allow
                     ))
                     logger.info('Secret {} does not have a resource policy, inserting a "stub" policy instead'.format(secret_arn))
 
-        except botocore.exceptions.ClientError as ex:
+        except (botocore.exceptions.ClientError, ConnectTimeoutError, EndpointConnectionError) as ex:
             logger.info('Unable to search Secrets Manager in region {} for secrets. The region may be disabled, or '
                         'the current principal may not be authorized to access the service. '
                         'Continuing.'.format(sm_region))

principalmapper/graphing/lambda_edges.py

@@ -20,7 +20,8 @@
 import os
 from typing import List, Optional
 
-from botocore.exceptions import ClientError
+from botocore.config import Config
+from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError
 
 from principalmapper.common import Edge, Node
 from principalmapper.graphing.edge_checker import EdgeChecker
@@ -51,7 +52,11 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]]
         if self.session is not None:
             lambda_regions = botocore_tools.get_regions_to_search(self.session, 'lambda', region_allow_list, region_deny_list)
             for region in lambda_regions:
-                lambda_clients.append(self.session.create_client('lambda', region_name=region, **lambdaargs))
+                lambda_clients.append(self.session.create_client(
+                    'lambda', region_name=region,
+                    config=Config(connect_timeout=10, retries={'max_attempts': 0}),
+                    **lambdaargs
+                ))
 
         # grab existing lambda functions
         function_list = []
@@ -61,8 +66,8 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]]
                 for page in paginator.paginate(PaginationConfig={'PageSize': 25}):
                     for func in page['Functions']:
                         function_list.append(func)
-            except ClientError as ex:
-                logger.warning('Unable to search region {} for stacks. The region may be disabled, or the error may '
+            except (ClientError, ConnectTimeoutError, EndpointConnectionError) as ex:
+                logger.warning('Unable to search region {} for functions. The region may be disabled, or the error may '
                                'be caused by an authorization issue. Continuing.'.format(lambda_client.meta.region_name))
                 logger.debug('Exception details: {}'.format(ex))
 

principalmapper/util/case_insensitive_dict.py

@@ -31,7 +31,8 @@
 #      See the License for the specific language governing permissions and
 #      limitations under the License.
 
-from collections import Mapping, MutableMapping, OrderedDict
+from collections import OrderedDict
+from collections.abc import Mapping, MutableMapping
 
 
 class CaseInsensitiveDict(MutableMapping):