Import Map

data_request_api/version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.1.3.dev19+g38e0b48.d20250407'
+__version_tuple__ = version_tuple = (1, 1, 3, 'dev19', 'g38e0b48.d20250407')

generate_importmap.ipynb

@@ -0,0 +1,363 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# !pip install python-scalpel\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from scalpel.import_graph.import_graph import ImportGraph, Tree"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Current working directory: /Users/daniel.ellis/WIPwork/CMIP7_DReq_Software\n",
+      "Target directory: /Users/daniel.ellis/WIPwork/CMIP7_DReq_Software/data_request_api\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "print(\"Current working directory:\", os.getcwd())\n",
+    "target_dir = os.getcwd()+'/data_request_api'  # Use the current working directory as the target directory\n",
+    "print(\"Target directory:\", target_dir)\n",
+    "import_graph = ImportGraph(target_dir)\n",
+    "import_graph.build_dir_tree()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'data_request_api.version': {'typing': ['Tuple', 'Union']},\n",
+       " 'data_request_api.data_request_api.__init__.py': {'version': ['*']},\n",
+       " 'data_request_api.data_request_api.tests.test_dreq_content': {'data_request_api.content': ['dreq_content'],\n",
+       "  'data_request_api.utilities.logger': ['change_log_file',\n",
+       "   'change_log_level']},\n",
+       " 'data_request_api.data_request_api.tests.test_config_cli': {'pathlib': ['Path']},\n",
+       " 'data_request_api.data_request_api.tests.test_optimize': {'__future__': ['print_function',\n",
+       "   'division',\n",
+       "   'unicode_literals',\n",
+       "   'absolute_import'],\n",
+       "  'data_request_api.query.data_request': ['DataRequest'],\n",
+       "  'data_request_api.content.dreq_content': ['_dreq_res'],\n",
+       "  'data_request_api.utilities.tools': ['read_json_input_file_content'],\n",
+       "  'data_request_api.content.dump_transformation': ['correct_dictionaries',\n",
+       "   'transform_content_inner',\n",
+       "   'get_transformed_content',\n",
+       "   'get_transform_settings'],\n",
+       "  'data_request_api.tests': ['filepath']},\n",
+       " 'data_request_api.data_request_api.tests.test_dump_transformation': {'__future__': ['print_function',\n",
+       "   'division',\n",
+       "   'unicode_literals',\n",
+       "   'absolute_import'],\n",
+       "  'data_request_api.utilities.tools': ['read_json_file'],\n",
+       "  'data_request_api.content.dump_transformation': ['correct_key_string',\n",
+       "   'correct_dictionaries',\n",
+       "   'transform_content_inner',\n",
+       "   'transform_content',\n",
+       "   'split_content_one_base',\n",
+       "   'get_transform_settings'],\n",
+       "  'data_request_api.tests': ['filepath']},\n",
+       " 'data_request_api.data_request_api.tests.test_consolidation': {'data_request_api.content': ['dreq_content'],\n",
+       "  'data_request_api.content.consolidate_export': ['_apply_consistency_fixes',\n",
+       "   '_filter_references',\n",
+       "   '_map_attribute',\n",
+       "   '_map_record_id',\n",
+       "   'map_data'],\n",
+       "  'data_request_api.content.mapping_table': ['version_consistency_drop_fields',\n",
+       "   'version_consistency_fields'],\n",
+       "  'data_request_api.tests': ['filepath'],\n",
+       "  'data_request_api.utilities.logger': ['change_log_file', 'change_log_level'],\n",
+       "  'data_request_api.utilities.tools': ['read_json_file',\n",
+       "   'write_json_output_file_content']},\n",
+       " 'data_request_api.data_request_api.tests.__init__.py': {},\n",
+       " 'data_request_api.data_request_api.tests.test_data_request': {'__future__': ['print_function',\n",
+       "   'division',\n",
+       "   'unicode_literals',\n",
+       "   'absolute_import'],\n",
+       "  'data_request_api.utilities.tools': ['read_json_input_file_content'],\n",
+       "  'data_request_api.query.data_request': ['DRObjects',\n",
+       "   'ExperimentsGroup',\n",
+       "   'VariablesGroup',\n",
+       "   'Opportunity',\n",
+       "   'DataRequest',\n",
+       "   'version'],\n",
+       "  'data_request_api.query.vocabulary_server': ['VocabularyServer',\n",
+       "   'ConstantValueObj'],\n",
+       "  'data_request_api.tests': ['filepath']},\n",
+       " 'data_request_api.data_request_api.tests.test_vocabulary_server': {'__future__': ['print_function',\n",
+       "   'division',\n",
+       "   'unicode_literals',\n",
+       "   'absolute_import'],\n",
+       "  'data_request_api.utilities.tools': ['read_json_input_file_content'],\n",
+       "  'data_request_api.query.vocabulary_server': ['VocabularyServer',\n",
+       "   'is_link_id_or_value',\n",
+       "   'build_link_from_id',\n",
+       "   'to_plural',\n",
+       "   'to_singular'],\n",
+       "  'data_request_api.tests': ['filepath']},\n",
+       " 'data_request_api.data_request_api.tests.test_config': {'pathlib': ['Path'],\n",
+       "  'data_request_api.utilities.config': ['DEFAULT_CONFIG',\n",
+       "   '_sanity_check',\n",
+       "   'load_config',\n",
+       "   'update_config']},\n",
+       " 'data_request_api.data_request_api.tests.test_cli': {'pathlib': ['Path']},\n",
+       " 'data_request_api.data_request_api.tests.test_decorator': {'data_request_api.utilities.decorators': ['append_kwargs_from_config'],\n",
+       "  'data_request_api.utilities.logger': ['change_log_file',\n",
+       "   'change_log_level']},\n",
+       " 'data_request_api.data_request_api.content.dump_transformation': {'__future__': ['division',\n",
+       "   'print_function',\n",
+       "   'unicode_literals',\n",
+       "   'absolute_import'],\n",
+       "  'collections': ['defaultdict'],\n",
+       "  'data_request_api.utilities.decorators': ['append_kwargs_from_config'],\n",
+       "  'data_request_api.utilities.logger': ['get_logger',\n",
+       "   'change_log_level',\n",
+       "   'change_log_file'],\n",
+       "  'data_request_api.utilities.tools': ['read_json_input_file_content',\n",
+       "   'write_json_output_file_content'],\n",
+       "  'data_request_api.content': ['dreq_content']},\n",
+       " 'data_request_api.data_request_api.content.consolidate_export': {'collections': ['defaultdict'],\n",
+       "  'data_request_api.utilities.logger': ['get_logger'],\n",
+       "  'mapping_table': ['version_consistency',\n",
+       "   'version_consistency_drop_fields',\n",
+       "   'version_consistency_drop_tables',\n",
+       "   'version_consistency_fields']},\n",
+       " 'data_request_api.data_request_api.content.__init__.py': {},\n",
+       " 'data_request_api.data_request_api.content.mapping_table': {},\n",
+       " 'data_request_api.data_request_api.content.dreq_content': {'filecmp': ['cmp'],\n",
+       "  'shutil': ['move'],\n",
+       "  'bs4': ['BeautifulSoup'],\n",
+       "  'data_request_api.content': ['consolidate_export'],\n",
+       "  'data_request_api.content.mapping_table': ['mapping_table'],\n",
+       "  'data_request_api.utilities.decorators': ['append_kwargs_from_config'],\n",
+       "  'data_request_api.utilities.logger': ['get_logger']},\n",
+       " 'data_request_api.data_request_api.utilities.config': {'importlib.metadata': ['version',\n",
+       "   'PackageNotFoundError'],\n",
+       "  'pathlib': ['Path']},\n",
+       " 'data_request_api.data_request_api.utilities.tools': {'__future__': ['division',\n",
+       "   'absolute_import',\n",
+       "   'print_function',\n",
+       "   'unicode_literals'],\n",
+       "  'data_request_api.utilities.logger': ['get_logger']},\n",
+       " 'data_request_api.data_request_api.utilities.__init__.py': {},\n",
+       " 'data_request_api.data_request_api.utilities.logger': {'__future__': ['unicode_literals',\n",
+       "   'print_function',\n",
+       "   'absolute_import',\n",
+       "   'division']},\n",
+       " 'data_request_api.data_request_api.utilities.parser': {},\n",
+       " 'data_request_api.data_request_api.utilities.decorators': {'data_request_api.utilities.config': ['_sanity_check'],\n",
+       "  'data_request_api.utilities.logger': ['get_logger']},\n",
+       " 'data_request_api.data_request_api.query.vocabulary_server': {'__future__': ['division',\n",
+       "   'print_function',\n",
+       "   'unicode_literals',\n",
+       "   'absolute_import'],\n",
+       "  'collections': ['defaultdict'],\n",
+       "  'data_request_api.utilities.logger': ['get_logger'],\n",
+       "  'data_request_api.utilities.tools': ['read_json_file']},\n",
+       " 'data_request_api.data_request_api.query.data_request': {'__future__': ['division',\n",
+       "   'print_function',\n",
+       "   'unicode_literals',\n",
+       "   'absolute_import'],\n",
+       "  'collections': ['defaultdict', 'namedtuple'],\n",
+       "  'itertools': ['product', 'chain'],\n",
+       "  'data_request_api.utilities.logger': ['get_logger',\n",
+       "   'change_log_file',\n",
+       "   'change_log_level'],\n",
+       "  'data_request_api.content.dump_transformation': ['transform_content'],\n",
+       "  'data_request_api.utilities.tools': ['read_json_file',\n",
+       "   'write_csv_output_file_content'],\n",
+       "  'data_request_api.query.vocabulary_server': ['VocabularyServer',\n",
+       "   'is_link_id_or_value',\n",
+       "   'build_link_from_id',\n",
+       "   'to_singular',\n",
+       "   'ConstantValueObj',\n",
+       "   'to_plural'],\n",
+       "  'data_request_api': ['version']},\n",
+       " 'data_request_api.data_request_api.query.__init__.py': {},\n",
+       " 'data_request_api.data_request_api.query.dreq_query': {'collections': ['OrderedDict'],\n",
+       "  'data_request_api.query.dreq_classes': ['DreqTable',\n",
+       "   'ExptRequest',\n",
+       "   'UNIQUE_VAR_NAME',\n",
+       "   'PRIORITY_LEVELS',\n",
+       "   'format_attribute_name'],\n",
+       "  'data_request_api.utilities.decorators': ['append_kwargs_from_config'],\n",
+       "  'data_request_api.utilities.tools': ['write_csv_output_file_content'],\n",
+       "  'data_request_api': ['version']},\n",
+       " 'data_request_api.data_request_api.query.dreq_classes': {'dataclasses': ['dataclass',\n",
+       "   'field'],\n",
+       "  'typing': ['Set']},\n",
+       " 'data_request_api.data_request_api.command_line.get_variables_metadata': {'data_request_api': ['version']},\n",
+       " 'data_request_api.data_request_api.command_line.config': {'data_request_api.utilities': ['config']},\n",
+       " 'data_request_api.data_request_api.command_line.__init__.py': {},\n",
+       " 'data_request_api.data_request_api.command_line.export_dreq_lists_json': {'collections': ['OrderedDict']},\n",
+       " 'data_request_api.data_request_api.command_line.compare_variables': {'collections': ['OrderedDict',\n",
+       "   'defaultdict']},\n",
+       " 'data_request_api.data_request_api.command_line.estimate_dreq_volume': {'collections': ['OrderedDict',\n",
+       "   'defaultdict']}}"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "all_leaf_nodes = import_graph.get_leaf_nodes()\n",
+    "\n",
+    "module_imports = {}\n",
+    "for node in all_leaf_nodes:\n",
+    "    module_dict = import_graph.parse_import(node.ast)\n",
+    "    module_imports[import_graph.leaf2root(node)] = module_dict\n",
+    "\n",
+    "module_imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Warning: using box for unknown shape rhombus\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Graph written to import_graph.dot and rendered to import_graph.pdf\n"
+     ]
+    }
+   ],
+   "source": [
+    "import json\n",
+    "from graphviz import Digraph\n",
+    "import os\n",
+    "\n",
+    "# Your import data\n",
+    "import_data = module_imports  # replace with your dict\n",
+    "\n",
+    "# Setup dot path\n",
+    "dot_path = \"/Users/daniel.ellis/customlib/homebrew/Caskroom/mambaforge/base/bin/\"\n",
+    "os.environ[\"PATH\"] = dot_path + os.pathsep + os.environ[\"PATH\"]\n",
+    "\n",
+    "dot = Digraph(comment=\"CMIPLD Module Import Graph\", engine=\"sfdp\")\n",
+    "dot.attr(rankdir='TB')\n",
+    "dot.attr(compound='true', splines='true', overlap='false', concentrate='true')\n",
+    "\n",
+    "def categorize_node(name):\n",
+    "    if name.endswith(\".py\"):\n",
+    "        return \"pyfile\"\n",
+    "    if name.startswith(\"cmipld.\"):\n",
+    "        level = name.count('.')\n",
+    "        if level == 1:\n",
+    "            return \"top-level\"\n",
+    "        else:\n",
+    "            return \"nested\"\n",
+    "    else:\n",
+    "        return \"external\"\n",
+    "\n",
+    "all_nodes = set()\n",
+    "target_nodes = set()\n",
+    "\n",
+    "for module, imports in import_data.items():\n",
+    "    all_nodes.add(module)\n",
+    "    for imp_module in imports.keys():\n",
+    "        if isinstance(imp_module, int):\n",
+    "            continue\n",
+    "        all_nodes.add(imp_module)\n",
+    "        target_nodes.add(imp_module)\n",
+    "\n",
+    "styles = {\n",
+    "    \"external\": {\"color\": \"blue\", \"shape\": \"box\", \"style\": \"filled\", \"fillcolor\": \"#c6dbef\"},\n",
+    "    \"nested\": {\"color\": \"darkgreen\", \"shape\": \"ellipse\", \"style\": \"filled\", \"fillcolor\": \"#a1d99b\"},\n",
+    "    \"top-level\": {\"color\": \"red\", \"shape\": \"diamond\", \"style\": \"filled\", \"fillcolor\": \"#fcbba1\"},\n",
+    "    \"isolated\": {\"color\": \"grey\", \"shape\": \"circle\", \"style\": \"filled\", \"fillcolor\": \"#eeeeee\"},\n",
+    "    \"pyfile\": {\"color\": \"orange\", \"shape\": \"rhombus\", \"style\": \"filled\", \"fillcolor\": \"#fdd0a2\"},\n",
+    "}\n",
+    "\n",
+    "for node in all_nodes:\n",
+    "    if not node:\n",
+    "        continue\n",
+    "    category = categorize_node(node)\n",
+    "    if node not in target_nodes and node not in import_data:\n",
+    "        category = \"isolated\"\n",
+    "    style = styles.get(category, {})\n",
+    "    dot.node(node,\n",
+    "             color=style.get(\"color\"),\n",
+    "             shape=style.get(\"shape\"),\n",
+    "             style=style.get(\"style\"),\n",
+    "             fillcolor=style.get(\"fillcolor\"))\n",
+    "\n",
+    "for module, imports in import_data.items():\n",
+    "    for imp_module, items in imports.items():\n",
+    "        if isinstance(imp_module, int):\n",
+    "            continue\n",
+    "        try:\n",
+    "            dot.edge(module, imp_module)\n",
+    "        except Exception as e:\n",
+    "            print(f\"Error adding edge from {module} to {imp_module}: {e}\")\n",
+    "\n",
+    "with open(\"import_graph.dot\", \"w\") as f:\n",
+    "    f.write(dot.source)\n",
+    "\n",
+    "dot.render(\"import_graph\", format=\"pdf\", cleanup=True)\n",
+    "print(\"Graph written to import_graph.dot and rendered to import_graph.pdf\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "temp",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}