[v2] Hide generate_presigned_url from docs and implement nested docs within vendored botocore

awscli/botocore/docs/__init__.py

@@ -25,15 +25,21 @@ def generate_docs(root_dir, session):
         service's reference documentation is loacated at
         root_dir/reference/services/service-name.rst
     """
-    services_doc_path = os.path.join(root_dir, 'reference', 'services')
-    if not os.path.exists(services_doc_path):
-        os.makedirs(services_doc_path)
+    # Create the root directory where all service docs live.
+    services_dir_path = os.path.join(root_dir, 'reference', 'services')
+    if not os.path.exists(services_dir_path):
+        os.makedirs(services_dir_path)
 
     # Generate reference docs and write them out.
     for service_name in session.get_available_services():
-        docs = ServiceDocumenter(service_name, session).document_service()
-        service_doc_path = os.path.join(
-            services_doc_path, service_name + '.rst'
+        docs = ServiceDocumenter(
+            service_name, session, services_dir_path
+        ).document_service()
+
+        # Write the main service documentation page.
+        # Path: <root>/reference/services/<service>/index.rst
+        service_file_path = os.path.join(
+            services_dir_path, f'{service_name}.rst'
         )
-        with open(service_doc_path, 'wb') as f:
+        with open(service_file_path, 'wb') as f:
             f.write(docs)

awscli/botocore/docs/bcdoc/restdoc.py

@@ -11,6 +11,7 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 import logging
+import os
 
 from botocore.compat import OrderedDict
 from botocore.docs.bcdoc.docstringparser import DocStringParser
@@ -219,3 +220,15 @@ def remove_all_sections(self):
 
     def clear_text(self):
         self._writes = []
+
+    def add_title_section(self, title):
+        title_section = self.add_new_section('title')
+        title_section.style.h1(title)
+        return title_section
+
+    def write_to_file(self, full_path, file_name):
+        if not os.path.exists(full_path):
+            os.makedirs(full_path)
+        sub_resource_file_path = os.path.join(full_path, f'{file_name}.rst')
+        with open(sub_resource_file_path, 'wb') as f:
+            f.write(self.flush_structure())

awscli/botocore/docs/client.py

@@ -11,8 +11,10 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 import inspect
+import os
 
 from botocore.compat import OrderedDict
+from botocore.docs.bcdoc.restdoc import DocumentStructure
 from botocore.docs.example import ResponseExampleDocumenter
 from botocore.docs.method import (
     document_custom_method,
@@ -24,9 +26,21 @@
 from botocore.docs.utils import DocumentedShape, get_official_service_name
 
 
+def _allowlist_generate_presigned_url(method_name, service_name, **kwargs):
+    if method_name != 'generate_presigned_url':
+        return None
+    return service_name in ['s3']
+
+
 class ClientDocumenter:
-    def __init__(self, client, shared_examples=None):
+    _CLIENT_METHODS_FILTERS = [
+        _allowlist_generate_presigned_url,
+    ]
+
+    def __init__(self, client, root_docs_path, shared_examples=None):
         self._client = client
+        self._client_class_name = self._client.__class__.__name__
+        self._root_docs_path = root_docs_path
         self._shared_examples = shared_examples
         if self._shared_examples is None:
             self._shared_examples = {}
@@ -39,9 +53,32 @@ def document_client(self, section):
         """
         self._add_title(section)
         self._add_class_signature(section)
-        client_methods = get_instance_public_methods(self._client)
+        client_methods = self._get_client_methods()
         self._add_client_intro(section, client_methods)
-        self._add_client_methods(section, client_methods)
+        self._add_client_methods(client_methods)
+
+    def _get_client_methods(self):
+        client_methods = get_instance_public_methods(self._client)
+        return self._filter_client_methods(client_methods)
+
+    def _filter_client_methods(self, client_methods):
+        filtered_methods = {}
+        for method_name, method in client_methods.items():
+            include = self._filter_client_method(
+                method=method,
+                method_name=method_name,
+                service_name=self._service_name,
+            )
+            if include:
+                filtered_methods[method_name] = method
+        return filtered_methods
+
+    def _filter_client_method(self, **kwargs):
+        for filter in self._CLIENT_METHODS_FILTERS:
+            filter_include = filter(**kwargs)
+            if filter_include is not None:
+                return filter_include
+        return True
 
     def _add_title(self, section):
         section.style.h2('Client')
@@ -64,16 +101,16 @@ def _add_client_intro(self, section, client_methods):
         self._add_client_creation_example(section)
 
         # List out all of the possible client methods.
-        section.style.new_line()
-        section.write('These are the available methods:')
-        section.style.new_line()
-        class_name = self._client.__class__.__name__
+        section.style.dedent()
+        section.style.new_paragraph()
+        section.writeln('These are the available methods:')
+        section.style.toctree()
         for method_name in sorted(client_methods):
-            section.style.li(f':py:meth:`~{class_name}.Client.{method_name}`')
+            section.style.tocitem(f'{self._service_name}/client/{method_name}')
 
     def _add_class_signature(self, section):
         section.style.start_sphinx_py_class(
-            class_name=f'{self._client.__class__.__name__}.Client'
+            class_name=f'{self._client_class_name}.Client'
         )
 
     def _add_client_creation_example(self, section):
@@ -84,19 +121,29 @@ def _add_client_creation_example(self, section):
         )
         section.style.end_codeblock()
 
-    def _add_client_methods(self, section, client_methods):
-        section = section.add_new_section('methods')
+    def _add_client_methods(self, client_methods):
         for method_name in sorted(client_methods):
+            # Create a new DocumentStructure for each client method and add contents.
+            method_doc_structure = DocumentStructure(
+                method_name, target='html'
+            )
             self._add_client_method(
-                section, method_name, client_methods[method_name]
+                method_doc_structure, method_name, client_methods[method_name]
+            )
+            # Write client methods in individual/nested files.
+            # Path: <root>/reference/services/<service>/client/<method_name>.rst
+            client_dir_path = os.path.join(
+                self._root_docs_path, self._service_name, 'client'
             )
+            method_doc_structure.write_to_file(client_dir_path, method_name)
 
     def _add_client_method(self, section, method_name, method):
-        section = section.add_new_section(method_name)
+        section.add_title_section(method_name)
+        method_section = section.add_new_section(method_name)
         if self._is_custom_method(method_name):
-            self._add_custom_method(section, method_name, method)
+            self._add_custom_method(method_section, method_name, method)
         else:
-            self._add_model_driven_method(section, method_name)
+            self._add_model_driven_method(method_section, method_name)
 
     def _is_custom_method(self, method_name):
         return method_name not in self._client.meta.method_to_api_mapping
@@ -109,9 +156,10 @@ def _add_method_exceptions_list(self, section, operation_model):
         error_section.style.new_line()
         error_section.style.bold('Exceptions')
         error_section.style.new_line()
-        client_name = self._client.__class__.__name__
         for error in operation_model.error_shapes:
-            class_name = f'{client_name}.Client.exceptions.{error.name}'
+            class_name = (
+                f'{self._client_class_name}.Client.exceptions.{error.name}'
+            )
             error_section.style.li(f':py:class:`{class_name}`')
 
     def _add_model_driven_method(self, section, method_name):
@@ -177,15 +225,18 @@ class ClientExceptionsDocumenter:
         ),
     )
 
-    def __init__(self, client):
+    def __init__(self, client, root_docs_path):
         self._client = client
+        self._client_class_name = self._client.__class__.__name__
+        self._service_name = self._client.meta.service_model.service_name
         self._service_id = self._client.meta.service_model.service_id
+        self._root_docs_path = root_docs_path
 
     def document_exceptions(self, section):
         self._add_title(section)
         self._add_overview(section)
         self._add_exceptions_list(section)
-        self._add_exception_classes(section)
+        self._add_exception_classes()
 
     def _add_title(self, section):
         section.style.h2('Client Exceptions')
@@ -207,7 +258,7 @@ def _add_overview(self, section):
 
     def _exception_class_name(self, shape):
         cls_name = self._client.__class__.__name__
-        return f'{cls_name}.Client.exceptions.{shape.name}'
+        return f'{self._client_class_name}.Client.exceptions.{shape.name}'
 
     def _add_exceptions_list(self, section):
         error_shapes = self._client.meta.service_model.error_shapes
@@ -217,17 +268,34 @@ def _add_exceptions_list(self, section):
             section.style.new_line()
             return
         section.style.new_line()
-        section.write('The available client exceptions are:')
-        section.style.new_line()
+        section.writeln('The available client exceptions are:')
+        section.style.toctree()
         for shape in error_shapes:
-            class_name = self._exception_class_name(shape)
-            section.style.li(f':py:class:`{class_name}`')
+            section.style.tocitem(
+                f'{self._service_name}/client/exceptions/{shape.name}'
+            )
 
-    def _add_exception_classes(self, section):
+    def _add_exception_classes(self):
         for shape in self._client.meta.service_model.error_shapes:
-            self._add_exception_class(section, shape)
+            # Create a new DocumentStructure for each exception method and add contents.
+            exception_doc_structure = DocumentStructure(
+                shape.name, target='html'
+            )
+            self._add_exception_class(exception_doc_structure, shape)
+            # Write exceptions in individual/nested files.
+            # Path: <root>/reference/services/<service>/client/exceptions/<exception_name>.rst
+            exception_dir_path = os.path.join(
+                self._root_docs_path,
+                self._service_name,
+                'client',
+                'exceptions',
+            )
+            exception_doc_structure.write_to_file(
+                exception_dir_path, shape.name
+            )
 
     def _add_exception_class(self, section, shape):
+        section.add_title_section(shape.name)
         class_section = section.add_new_section(shape.name)
         class_name = self._exception_class_name(shape)
         class_section.style.start_sphinx_py_class(class_name=class_name)

awscli/botocore/docs/paginator.py

@@ -10,67 +10,106 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+import os
+
 from botocore import xform_name
 from botocore.compat import OrderedDict
+from botocore.docs.bcdoc.restdoc import DocumentStructure
 from botocore.docs.method import document_model_driven_method
 from botocore.docs.utils import DocumentedShape
 from botocore.utils import get_service_module_name
 
 
 class PaginatorDocumenter:
-    def __init__(self, client, service_paginator_model):
+    def __init__(self, client, service_paginator_model, root_docs_path):
         self._client = client
+        self._client_class_name = self._client.__class__.__name__
         self._service_name = self._client.meta.service_model.service_name
         self._service_paginator_model = service_paginator_model
+        self._root_docs_path = root_docs_path
+        self._USER_GUIDE_LINK = (
+            'https://boto3.amazonaws.com/'
+            'v1/documentation/api/latest/guide/paginators.html'
+        )
 
     def document_paginators(self, section):
         """Documents the various paginators for a service
 
         param section: The section to write to.
         """
         section.style.h2('Paginators')
+        self._add_overview(section)
         section.style.new_line()
         section.writeln('The available paginators are:')
+        section.style.toctree()
 
         paginator_names = sorted(
             self._service_paginator_model._paginator_config
         )
 
         # List the available paginators and then document each paginator.
         for paginator_name in paginator_names:
-            section.style.li(
-                f':py:class:`{self._client.__class__.__name__}.Paginator.{paginator_name}`'
+            section.style.tocitem(
+                f'{self._service_name}/paginator/{paginator_name}'
+            )
+            # Create a new DocumentStructure for each paginator and add contents.
+            paginator_doc_structure = DocumentStructure(
+                paginator_name, target='html'
+            )
+            self._add_paginator(paginator_doc_structure, paginator_name)
+            # Write paginators in individual/nested files.
+            # Path: <root>/reference/services/<service>/paginator/<paginator_name>.rst
+            paginator_dir_path = os.path.join(
+                self._root_docs_path, self._service_name, 'paginator'
+            )
+            paginator_doc_structure.write_to_file(
+                paginator_dir_path, paginator_name
             )
-            self._add_paginator(section, paginator_name)
 
     def _add_paginator(self, section, paginator_name):
-        section = section.add_new_section(paginator_name)
+        section.add_title_section(paginator_name)
 
         # Docment the paginator class
-        section.style.start_sphinx_py_class(
-            class_name=f'{self._client.__class__.__name__}.Paginator.{paginator_name}'
+        paginator_section = section.add_new_section(paginator_name)
+        paginator_section.style.start_sphinx_py_class(
+            class_name=f'{self._client_class_name}.Paginator.{paginator_name}'
         )
-        section.style.start_codeblock()
-        section.style.new_line()
+        paginator_section.style.start_codeblock()
+        paginator_section.style.new_line()
 
         # Document how to instantiate the paginator.
-        section.write(
+        paginator_section.write(
             f'paginator = client.get_paginator(\'{xform_name(paginator_name)}\')'
         )
-        section.style.end_codeblock()
-        section.style.new_line()
+        paginator_section.style.end_codeblock()
+        paginator_section.style.new_line()
         # Get the pagination model for the particular paginator.
         paginator_config = self._service_paginator_model.get_paginator(
             paginator_name
         )
         document_paginate_method(
-            section=section,
+            section=paginator_section,
             paginator_name=paginator_name,
             event_emitter=self._client.meta.events,
             service_model=self._client.meta.service_model,
             paginator_config=paginator_config,
         )
 
+    def _add_overview(self, section):
+        section.style.new_line()
+        section.write(
+            'Paginators are available on a client instance '
+            'via the ``get_paginator`` method. For more detailed instructions '
+            'and examples on the usage of paginators, see the '
+            'paginators '
+        )
+        section.style.external_link(
+            title='user guide',
+            link=self._USER_GUIDE_LINK,
+        )
+        section.write('.')
+        section.style.new_line()
+
 
 def document_paginate_method(
     section,