switch to modulegraph 2

.gitignore

@@ -44,6 +44,8 @@ __pycache__/
 /tests/python_exe.build
 /tests/report.xml
 
+/tests/unit/test_modulegraph2/**/*.pyd
+
 /tests/**/build/
 /tests/**/dist/
 

PyInstaller/building/toc_conversion.py

@@ -41,7 +41,7 @@ def __init__(self, graph, additional_files):
         # Include files that were found by hooks.
         # graph.iter_graph() should include only those modules that are reachable
         # from top-level script.
-        for node in graph.iter_graph(start=graph._top_script_node):
+        for node in graph.iter_graph(node=graph._top_script_node):
             # Update 'binaries', 'datas'
             name = node.identifier
             if name in additional_files:

PyInstaller/depend/analysis.py

@@ -56,7 +56,8 @@
         PURE_PYTHON_MODULE_TYPES, BINARY_MODULE_TYPES, VALID_MODULE_TYPES, \
         BAD_MODULE_TYPES, MODULE_TYPES_TO_TOC_DICT
 from ..lib.modulegraph.find_modules import get_implies
-from ..lib.modulegraph.modulegraph import ModuleGraph
+from ..lib.modulegraph2 import ModuleGraph
+from ..lib.modulegraph2._modulegraph import DEFAULT_DEPENDENCY
 from ..utils.hooks import collect_submodules, is_package
 
 
@@ -106,19 +107,25 @@ class PyiModuleGraph(ModuleGraph):
     # Note: these levels are completely arbitrary and may be adjusted if needed.
     LOG_LEVEL_MAPPING = {0: INFO, 1: DEBUG, 2: TRACE, 3: TRACE, 4: TRACE}
 
-    def __init__(self, pyi_homepath, user_hook_dirs=(), excludes=(), **kwargs):
-        super(PyiModuleGraph, self).__init__(excludes=excludes, **kwargs)
+    def __init__(self, pyi_homepath, user_hook_dirs=(), excludes=(), implies={}, **kwargs):
+        super(PyiModuleGraph, self).__init__(**kwargs)
+        self.add_excludes(excludes)
+        self.add_implies(implies)
         # Homepath to the place where is PyInstaller located.
         self._homepath = pyi_homepath
         # modulegraph Node for the main python script that is analyzed
         # by PyInstaller.
         self._top_script_node = None
+        self.path = []
 
         # Absolute paths of all user-defined hook directories.
         self._excludes = excludes
         self._reset(user_hook_dirs)
         self._analyze_base_modules()
 
+    def set_setuptools_nspackages(self):
+        pass
+
     def _reset(self, user_hook_dirs):
         """
         Reset for another set of scripts.
@@ -309,7 +316,7 @@ def add_script(self, pathname, caller=None):
                 sys.exit(1)
             # Create references from the top script to the base_modules in graph.
             for node in self._base_modules:
-                self.add_edge(self._top_script_node, node)
+                self.add_edge(self._top_script_node, node, DEFAULT_DEPENDENCY)
             # Return top-level script node.
             return self._top_script_node
         else:
@@ -318,7 +325,7 @@ def add_script(self, pathname, caller=None):
                 # top-level script.
                 caller = self._top_script_node
             return super(PyiModuleGraph, self).add_script(
-                pathname, caller=caller)
+                pathname)
 
 
     def process_post_graph_hooks(self):
@@ -348,7 +355,7 @@ def process_post_graph_hooks(self):
             for module_name, module_hooks in self._hooks.items():
                 # Graph node for this module if imported or "None" otherwise.
                 module_node = self.find_node(
-                    module_name, create_nspkg=False)
+                    module_name)
 
                 # If this module has not been imported, temporarily ignore it.
                 # This module is retained in the cache, as a subsequently run
@@ -499,7 +506,7 @@ def get_code_objects(self):
         """
         code_dict = {}
         mod_types = PURE_PYTHON_MODULE_TYPES
-        for node in self.iter_graph(start=self._top_script_node):
+        for node in self.iter_graph(node=self._top_script_node):
             # TODO This is terrible. To allow subclassing, types should never be
             # directly compared. Use isinstance() instead, which is safer,
             # simpler, and accepts sets. Most other calls to type() in the
@@ -534,7 +541,7 @@ def _make_toc(self, typecode=None, existing_TOC=None):
         module_filter = re.compile(regex_str)
 
         result = existing_TOC or TOC()
-        for node in self.iter_graph(start=self._top_script_node):
+        for node in self.iter_graph(node=self._top_script_node):
             # Skip modules that are in base_library.zip.
             if module_filter.match(node.identifier):
                 continue
@@ -625,6 +632,15 @@ def is_a_builtin(self, name) :
             return False
         return type(node).__name__ == 'BuiltinModule'
 
+    def flatten(self):
+        return self.iter_graph()
+
+    def import_hook(self, name, caller=None):
+        return self.iter_graph(node=self.add_module(name))
+
+    def find_node(self, node, create_nspkg=False):
+        return super().find_node(node)
+
     def get_importers(self, name):
         """List all modules importing the module with the passed name.
 
@@ -718,7 +734,7 @@ def add_hiddenimports(self, module_list):
             # Create references from the top script to the hidden import,
             # even if found otherwise. Don't waste time checking whether it
             # as actually added by this (test-) script.
-            self.add_edge(self._top_script_node, node)
+            self.add_edge(self._top_script_node, node, DEFAULT_DEPENDENCY)
 
 
     def get_co_using_ctypes(self):
@@ -740,7 +756,7 @@ def get_co_using_ctypes(self):
         node = self.find_node('ctypes')
         if node:
             referers = self.incoming(node)
-            for r in referers:
+            for (_, r) in referers:
                 # Under python 3.7 and earlier, if ctypes is added to
                 # hidden imports, one of referers ends up being None,
                 # causing #3825. Work around it.

PyInstaller/depend/imphook.py

@@ -486,7 +486,7 @@ def _process_hook_func(self):
             # Remove the graph link between the hooked module and item.
             # This removes the 'item' node from the graph if no other
             # links go to it (no other modules import it)
-            self.module_graph.removeReference(
+            self.module_graph.remove_all_edges(
                 hook_api.node, deleted_module_name)
 
 
@@ -569,7 +569,7 @@ def find_all_package_nodes(name):
         # TODO: Optimize this by using a pattern and walking the graph
         # only once.
         for item in set(self.excludedimports):
-            excluded_node = self.module_graph.find_node(item, create_nspkg=False)
+            excluded_node = self.module_graph.find_node(item)
             if excluded_node is None:
                 logger.info("Import to be excluded not found: %r", item)
                 continue
@@ -585,12 +585,12 @@ def find_all_package_nodes(name):
                 # modules, this `src` does import
                 references = set(
                     node.identifier
-                    for node in self.module_graph.outgoing(src))
+                    for (_, node) in self.module_graph.outgoing(src))
 
                 # Remove all of these imports which are also in
                 # "imports_to_remove".
                 for dest in imports_to_remove & references:
-                    self.module_graph.removeReference(src, dest)
+                    self.module_graph.remove_all_edges(src, dest)
                     logger.debug(
                         "Excluding import of %s from module %s", dest, src)
 

PyInstaller/depend/imphookapi.py

@@ -448,7 +448,7 @@ def imports(self):
         """
         List of the graph nodes of all modules directly imported by this module.
         """
-        return self.module_graph.iter_graph(start=self.module)
+        return self.module_graph.iter_graph(node=self.module)
 
     def add_imports(self, *module_names):
         """

PyInstaller/hooks/hook-sqlalchemy.py

@@ -62,7 +62,7 @@ def hook(hook_api):
 
     hidden_imports_set = set()
     known_imports = set()
-    for node in hook_api.module_graph.iter_graph(start=hook_api.module):
+    for node in hook_api.module_graph.iter_graph(node=hook_api.module):
         if isinstance(node, SourceModule) and \
                 node.identifier.startswith('sqlalchemy.'):
             known_imports.add(node.identifier)

PyInstaller/lib/modulegraph2/__init__.py

@@ -0,0 +1,65 @@
+"""
+A module import dependency graph for Python projects.
+
+This package defines a class representing the
+dependency graph between a collection of python
+modules and scripts, as well as supporting functions
+and classes.
+
+The graph itself is an subclass of :class:`objectgraph.ObjectGraph`.
+
+This module provides annotation for use with
+`Mypy <https://mypy.readthedocs.io/en/latest/>`_.
+"""
+__version__ = "2.1"
+
+from ._depinfo import DependencyInfo
+from ._distributions import PyPIDistribution, all_distributions, distribution_named
+from ._implies import Alias, Virtual
+from ._modulegraph import ModuleGraph
+from ._nodes import (
+    AliasNode,
+    BaseNode,
+    BuiltinModule,
+    BytecodeModule,
+    ExcludedModule,
+    ExtensionModule,
+    FrozenModule,
+    InvalidModule,
+    InvalidRelativeImport,
+    MissingModule,
+    Module,
+    NamespacePackage,
+    Package,
+    Script,
+    SourceModule,
+    VirtualNode,
+)
+from ._utilities import saved_sys_path
+
+__all__ = (
+    "Alias",
+    "AliasNode",
+    "BaseNode",
+    "BuiltinModule",
+    "BytecodeModule",
+    "DependencyInfo",
+    "ExcludedModule",
+    "ExtensionModule",
+    "FrozenModule",
+    "InvalidModule",
+    "InvalidRelativeImport",
+    "MissingModule",
+    "Module",
+    "ModuleGraph",
+    "NamespacePackage",
+    "Package",
+    "PyPIDistribution",
+    "Script",
+    "SourceModule",
+    "all_distributions",
+    "distribution_named",
+    "saved_sys_path",
+    "Virtual",
+    "VirtualNode",
+)