feat: add summary page for Classes and Modules (#361)

* feat: add summary page for Classes and Modules

* fix: update toc yaml alias to just be yaml alias

* feat: address review comments

Author: dandhlee

docfx_yaml/extension.py

@@ -28,7 +28,7 @@
 import logging
 
 from collections import defaultdict
-from collections.abc import MutableSet
+from collections.abc import MutableSet, Mapping, Sequence
 from pathlib import Path
 from functools import partial
 from itertools import zip_longest
@@ -134,6 +134,21 @@ class Bcolors:
     DEPRECATED: 'deprecated',
 }
 
+_SUMMARY_TYPE_BY_ITEM_TYPE = {
+    # Modules and Classes are similar types.
+    MODULE: CLASS,
+    CLASS: CLASS,
+}
+# Construct a mapping of name and content for each unique summary type entry.
+_ENTRY_NAME_AND_ENTRY_CONTENT_BY_SUMMARY_TYPE = {
+    summary_type: [[], []]  # entry name then entry content
+    for summary_type in set(_SUMMARY_TYPE_BY_ITEM_TYPE.values())
+}
+# Mapping for each summary page entry's file name and entry name.
+_FILE_NAME_AND_ENTRY_NAME_BY_SUMMARY_TYPE = {
+    CLASS: ('summary_class.yml', "Classes"),
+}
+
 # Disable blib2to3 output that clutters debugging log.
 logging.getLogger("blib2to3").setLevel(logging.ERROR)
 
@@ -1348,6 +1363,61 @@ def pretty_package_name(package_group):
     return " ".join(capitalized_name)
 
 
+# Type alias used for yaml entries.
+_yaml_type_alias = dict[str, any]
+
+
+def _find_and_add_summary_details(
+    yaml_data: _yaml_type_alias,
+    summary_type: str,
+    cgc_url: str,
+) -> None:
+    """Finds the summary details to add for a given entry."""
+    uid = yaml_data.get("uid", "")
+    item_to_add = uid if summary_type == CLASS else f"{uid}-summary"
+
+    if item_to_add not in _ENTRY_NAME_AND_ENTRY_CONTENT_BY_SUMMARY_TYPE[summary_type][0]:
+        _ENTRY_NAME_AND_ENTRY_CONTENT_BY_SUMMARY_TYPE[summary_type][0].append(
+            item_to_add
+        )
+
+    if summary_type in [CLASS]:
+        name_to_use = f"[{uid}]({cgc_url}{uid})"
+
+    _ENTRY_NAME_AND_ENTRY_CONTENT_BY_SUMMARY_TYPE[summary_type][1].append({
+        "uid": uid,
+        "name": name_to_use,
+        "fullName": uid,
+        "isExternal": False,
+    })
+
+
+def _render_summary_content(
+    children_name_and_summary_content: Sequence[Sequence[str | _yaml_type_alias]],
+    entry_name: str,
+    summary_type: str,
+    library_name: str,
+) -> _yaml_type_alias:
+    """Returns the summary content in appropriate YAML format to write."""
+    summary_content = {}
+
+    if summary_type in [CLASS]:
+        summary_content = {
+            "items": [{
+                'uid': f'{summary_type.lower()}-summary',
+                'name': entry_name,
+                'fullName': f'{entry_name} Summary',
+                'langs': ['python'],
+                'type': 'package',
+                'summary': f'Summary of entries of {entry_name} for {library_name}.',
+                'children': children_name_and_summary_content[0],
+            }],
+            'references': children_name_and_summary_content[1],
+        }
+
+    return summary_content
+
+
 def find_uid_to_convert(
     current_word: str,
     words: List[str],
@@ -1575,14 +1645,11 @@ def search_cross_references(obj, current_object_name: str, known_uids: List[str]
                     markdown_utils.reformat_markdown_to_html(attribute_type))
 
 
-# Type alias used for toc_yaml entries.
-_toc_yaml_type_alias = dict[str, any]
-
 def merge_markdown_and_package_toc(
-    pkg_toc_yaml: list[_toc_yaml_type_alias],
-    markdown_toc_yaml: _toc_yaml_type_alias,
+    pkg_toc_yaml: list[_yaml_type_alias],
+    markdown_toc_yaml: _yaml_type_alias,
     known_uids: set[str],
-) -> tuple[MutableSet[str], list[_toc_yaml_type_alias]]:
+) -> tuple[MutableSet[str], list[_yaml_type_alias]]:
     """
     Merges the markdown and package table of contents.
 
@@ -1595,8 +1662,8 @@ def merge_markdown_and_package_toc(
         contents file, with files in the correct position.
     """
     def _flatten_toc(
-        toc_yaml_entry: list[_toc_yaml_type_alias],
-    ) -> list[_toc_yaml_type_alias]:
+        toc_yaml_entry: list[_yaml_type_alias],
+    ) -> list[_yaml_type_alias]:
         """Flattens and retrieves all children within pkg_toc_yaml."""
         entries = list(toc_yaml_entry)
         for entry in toc_yaml_entry:
@@ -1897,6 +1964,17 @@ def convert_module_to_package_if_needed(obj):
         markdown_utils.remove_unused_pages(
             added_pages, app.env.moved_markdown_pages, normalized_outdir)
 
+    # Add summary pages as the second entry into the table of contents.
+    pkg_toc_yaml.insert(
+        1,
+        {
+            "name": f"{app.config.project} APIs",
+            "items": [
+                {"name": "Classes", "href": "summary_class.yml"},
+            ],
+        }
+    )
+
     toc_file = os.path.join(normalized_outdir, 'toc.yml')
     with open(toc_file, 'w') as writable:
         writable.write(
@@ -1909,6 +1987,11 @@ def convert_module_to_package_if_needed(obj):
             )
         )
 
+    cgc_url = (
+        "https://cloud.google.com/python/docs/reference/"
+        f"{app.config.project}/latest/"
+    )
+    yaml_entry_line = "### YamlMime:UniversalReference\n"
     # Output files
     for uid, data in iter(yaml_map.items()):
 
@@ -1935,7 +2018,7 @@ def convert_module_to_package_if_needed(obj):
             app.info(bold('[docfx_yaml] ') + darkgreen('Outputting %s' % filename))
 
         with open(out_file, 'w') as out_file_obj:
-            out_file_obj.write('### YamlMime:UniversalReference\n')
+            out_file_obj.write(yaml_entry_line)
             try:
                 dump(
                     {
@@ -1951,6 +2034,39 @@ def convert_module_to_package_if_needed(obj):
 
         file_name_set.add(filename)
 
+        for entry in yaml_data:
+            summary_type = _SUMMARY_TYPE_BY_ITEM_TYPE.get(entry.get("type"))
+            if not (summary_type):
+              continue
+
+            _find_and_add_summary_details(entry, summary_type, cgc_url)
+
+    for summary_type in _ENTRY_NAME_AND_ENTRY_CONTENT_BY_SUMMARY_TYPE:
+        children_names_and_content = (
+            _ENTRY_NAME_AND_ENTRY_CONTENT_BY_SUMMARY_TYPE[summary_type]
+        )
+        if not all(children_names_and_content):
+            continue
+
+        file_name, entry_name = _FILE_NAME_AND_ENTRY_NAME_BY_SUMMARY_TYPE[summary_type]
+
+        dump_content = _render_summary_content(
+            children_names_and_content,
+            entry_name,
+            summary_type,
+            app.config.project,
+        )
+
+        file_path_to_use = os.path.join(normalized_outdir, file_name)
+        with open(file_path_to_use, "w") as summary_file_obj:
+            summary_file_obj.write(yaml_entry_line)
+            dump(
+                dump_content,
+                summary_file_obj,
+                default_flow_style=False,
+            )
+
+
     index_file = os.path.join(normalized_outdir, 'index.yml')
     index_children = []
     index_references = []

docfx_yaml/markdown_utils.py

@@ -280,6 +280,7 @@ def move_markdown_pages(
 
         "reference.md", # Reference docs overlap with Overview. Will try and incorporate this in later.
                         # See https://github.com/googleapis/sphinx-docfx-yaml/issues/106.
+        "summary_overview.md",  # Already included in the TOC.
     ]
 
     files_to_rename = {