Skip to content

Commit 157f271

Browse files
FFY00ncoghlanbrettcannon
authored
gh-139899: Introduce MetaPathFinder.discover and PathEntryFinder.discover (#139900)
* gh-139899: Introduce MetaPathFinder.discover and PathEntryFinder.discover Signed-off-by: Filipe Laíns <lains@riseup.net> * Fix doc reference Signed-off-by: Filipe Laíns <lains@riseup.net> * Remove specific doc references Signed-off-by: Filipe Laíns <lains@riseup.net> * Fix docstrings Signed-off-by: Filipe Laíns <lains@riseup.net> * Revert "Remove specific doc references" This reverts commit 31d1a8f. Signed-off-by: Filipe Laíns <lains@riseup.net> * Fix news references Signed-off-by: Filipe Laíns <lains@riseup.net> * Add docs warning Signed-off-by: Filipe Laíns <lains@riseup.net> * Raise ValueError on invalid parent Signed-off-by: Filipe Laíns <lains@riseup.net> * Dedupe __path__ in PathFinder.discover Signed-off-by: Filipe Laíns <lains@riseup.net> * Use context manager and add error handling to os.scandir Signed-off-by: Filipe Laíns <lains@riseup.net> * Raise ValueError on invalid parent Signed-off-by: Filipe Laíns <lains@riseup.net> * Dedupe when package exists with multiple suffixes Signed-off-by: Filipe Laíns <lains@riseup.net> * Apply suggestions from code review Co-authored-by: Alyssa Coghlan <ncoghlan@gmail.com> * Add tests Signed-off-by: Filipe Laíns <lains@riseup.net> --------- Signed-off-by: Filipe Laíns <lains@riseup.net> Co-authored-by: Alyssa Coghlan <ncoghlan@gmail.com> Co-authored-by: Brett Cannon <brett@python.org>
1 parent f1cf762 commit 157f271

File tree

5 files changed

+235
-0
lines changed

5 files changed

+235
-0
lines changed

‎Doc/library/importlib.rst‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,28 @@ ABC hierarchy::
275275
.. versionchanged:: 3.4
276276
Returns ``None`` when called instead of :data:`NotImplemented`.
277277

278+
.. method:: discover(parent=None)
279+
280+
An optional method which searches for possible specs with given *parent*
281+
module spec. If *parent* is *None*, :meth:`MetaPathFinder.discover` will
282+
search for top-level modules.
283+
284+
Returns an iterable of possible specs.
285+
286+
Raises :exc:`ValueError` if *parent* is not a package module.
287+
288+
.. warning::
289+
This method can potentially yield a very large number of objects, and
290+
it may carry out IO operations when computing these values.
291+
292+
Because of this, it will generaly be desirable to compute the result
293+
values on-the-fly, as they are needed. As such, the returned object is
294+
only guaranteed to be an :class:`iterable <collections.abc.Iterable>`,
295+
instead of a :class:`list` or other
296+
:class:`collection <collections.abc.Collection>` type.
297+
298+
.. versionadded:: next
299+
278300

279301
.. class:: PathEntryFinder
280302

@@ -307,6 +329,28 @@ ABC hierarchy::
307329
:meth:`importlib.machinery.PathFinder.invalidate_caches`
308330
when invalidating the caches of all cached finders.
309331

332+
.. method:: discover(parent=None)
333+
334+
An optional method which searches for possible specs with given *parent*
335+
module spec. If *parent* is *None*, :meth:`PathEntryFinder.discover` will
336+
search for top-level modules.
337+
338+
Returns an iterable of possible specs.
339+
340+
Raises :exc:`ValueError` if *parent* is not a package module.
341+
342+
.. warning::
343+
This method can potentially yield a very large number of objects, and
344+
it may carry out IO operations when computing these values.
345+
346+
Because of this, it will generaly be desirable to compute the result
347+
values on-the-fly, as they are needed. As such, the returned object is
348+
only guaranteed to be an :class:`iterable <collections.abc.Iterable>`,
349+
instead of a :class:`list` or other
350+
:class:`collection <collections.abc.Collection>` type.
351+
352+
.. versionadded:: next
353+
310354

311355
.. class:: Loader
312356

‎Lib/importlib/_bootstrap_external.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1283,6 +1283,23 @@ def find_spec(cls, fullname, path=None, target=None):
12831283
else:
12841284
return spec
12851285

1286+
@classmethod
1287+
def discover(cls, parent=None):
1288+
if parent is None:
1289+
path = sys.path
1290+
elif parent.submodule_search_locations is None:
1291+
raise ValueError(f'{parent} is not a package module')
1292+
else:
1293+
path = parent.submodule_search_locations
1294+
1295+
for entry in set(path):
1296+
if not isinstance(entry, str):
1297+
continue
1298+
if (finder := cls._path_importer_cache(entry)) is None:
1299+
continue
1300+
if discover := getattr(finder, 'discover', None):
1301+
yield from discover(parent)
1302+
12861303
@staticmethod
12871304
def find_distributions(*args, **kwargs):
12881305
"""
@@ -1432,6 +1449,37 @@ def path_hook_for_FileFinder(path):
14321449

14331450
return path_hook_for_FileFinder
14341451

1452+
def _find_children(self):
1453+
with _os.scandir(self.path) as scan_iterator:
1454+
while True:
1455+
try:
1456+
entry = next(scan_iterator)
1457+
if entry.name == _PYCACHE:
1458+
continue
1459+
# packages
1460+
if entry.is_dir() and '.' not in entry.name:
1461+
yield entry.name
1462+
# files
1463+
if entry.is_file():
1464+
yield from {
1465+
entry.name.removesuffix(suffix)
1466+
for suffix, _ in self._loaders
1467+
if entry.name.endswith(suffix)
1468+
}
1469+
except OSError:
1470+
pass # ignore exceptions from next(scan_iterator) and os.DirEntry
1471+
except StopIteration:
1472+
break
1473+
1474+
def discover(self, parent=None):
1475+
if parent and parent.submodule_search_locations is None:
1476+
raise ValueError(f'{parent} is not a package module')
1477+
1478+
module_prefix = f'{parent.name}.' if parent else ''
1479+
for child_name in self._find_children():
1480+
if spec := self.find_spec(module_prefix + child_name):
1481+
yield spec
1482+
14351483
def __repr__(self):
14361484
return f'FileFinder({self.path!r})'
14371485

‎Lib/importlib/abc.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,16 @@ def invalidate_caches(self):
4545
This method is used by importlib.invalidate_caches().
4646
"""
4747

48+
def discover(self, parent=None):
49+
"""An optional method which searches for possible specs with given *parent*
50+
module spec. If *parent* is *None*, MetaPathFinder.discover will search
51+
for top-level modules.
52+
53+
Returns an iterable of possible specs.
54+
"""
55+
return ()
56+
57+
4858
_register(MetaPathFinder, machinery.BuiltinImporter, machinery.FrozenImporter,
4959
machinery.PathFinder, machinery.WindowsRegistryFinder)
5060

@@ -58,6 +68,15 @@ def invalidate_caches(self):
5868
This method is used by PathFinder.invalidate_caches().
5969
"""
6070

71+
def discover(self, parent=None):
72+
"""An optional method which searches for possible specs with given
73+
*parent* module spec. If *parent* is *None*, PathEntryFinder.discover
74+
will search for top-level modules.
75+
76+
Returns an iterable of possible specs.
77+
"""
78+
return ()
79+
6180
_register(PathEntryFinder, machinery.FileFinder)
6281

6382

‎Lib/test/test_importlib/test_discover.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
from unittest.mock import Mock
2+
3+
from test.test_importlib import util
4+
5+
importlib = util.import_importlib('importlib')
6+
machinery = util.import_importlib('importlib.machinery')
7+
8+
9+
class DiscoverableFinder:
10+
def __init__(self, discover=[]):
11+
self._discovered_values = discover
12+
13+
def find_spec(self, fullname, path=None, target=None):
14+
raise NotImplemented
15+
16+
def discover(self, parent=None):
17+
yield from self._discovered_values
18+
19+
20+
class TestPathFinder:
21+
"""PathFinder implements MetaPathFinder, which uses the PathEntryFinder(s)
22+
registered in sys.path_hooks (and sys.path_importer_cache) to search
23+
sys.path or the parent's __path__.
24+
25+
PathFinder.discover() should redirect to the .discover() method of the
26+
PathEntryFinder for each path entry.
27+
"""
28+
29+
def test_search_path_hooks_top_level(self):
30+
modules = [
31+
self.machinery.ModuleSpec(name='example1', loader=None),
32+
self.machinery.ModuleSpec(name='example2', loader=None),
33+
self.machinery.ModuleSpec(name='example3', loader=None),
34+
]
35+
36+
with util.import_state(
37+
path_importer_cache={
38+
'discoverable': DiscoverableFinder(discover=modules),
39+
},
40+
path=['discoverable'],
41+
):
42+
discovered = list(self.machinery.PathFinder.discover())
43+
44+
self.assertEqual(discovered, modules)
45+
46+
47+
def test_search_path_hooks_parent(self):
48+
parent = self.machinery.ModuleSpec(name='example', loader=None, is_package=True)
49+
parent.submodule_search_locations.append('discoverable')
50+
51+
children = [
52+
self.machinery.ModuleSpec(name='example.child1', loader=None),
53+
self.machinery.ModuleSpec(name='example.child2', loader=None),
54+
self.machinery.ModuleSpec(name='example.child3', loader=None),
55+
]
56+
57+
with util.import_state(
58+
path_importer_cache={
59+
'discoverable': DiscoverableFinder(discover=children)
60+
},
61+
path=[],
62+
):
63+
discovered = list(self.machinery.PathFinder.discover(parent))
64+
65+
self.assertEqual(discovered, children)
66+
67+
def test_invalid_parent(self):
68+
parent = self.machinery.ModuleSpec(name='example', loader=None)
69+
with self.assertRaises(ValueError):
70+
list(self.machinery.PathFinder.discover(parent))
71+
72+
73+
(
74+
Frozen_TestPathFinder,
75+
Source_TestPathFinder,
76+
) = util.test_both(TestPathFinder, importlib=importlib, machinery=machinery)
77+
78+
79+
class TestFileFinder:
80+
"""FileFinder implements PathEntryFinder and provides the base finder
81+
implementation to search the file system.
82+
"""
83+
84+
def get_finder(self, path):
85+
loader_details = [
86+
(self.machinery.SourceFileLoader, self.machinery.SOURCE_SUFFIXES),
87+
(self.machinery.SourcelessFileLoader, self.machinery.BYTECODE_SUFFIXES),
88+
]
89+
return self.machinery.FileFinder(path, *loader_details)
90+
91+
def test_discover_top_level(self):
92+
modules = {'example1', 'example2', 'example3'}
93+
with util.create_modules(*modules) as mapping:
94+
finder = self.get_finder(mapping['.root'])
95+
discovered = list(finder.discover())
96+
self.assertEqual({spec.name for spec in discovered}, modules)
97+
98+
def test_discover_parent(self):
99+
modules = {
100+
'example.child1',
101+
'example.child2',
102+
'example.child3',
103+
}
104+
with util.create_modules(*modules) as mapping:
105+
example = self.get_finder(mapping['.root']).find_spec('example')
106+
finder = self.get_finder(example.submodule_search_locations[0])
107+
discovered = list(finder.discover(example))
108+
self.assertEqual({spec.name for spec in discovered}, modules)
109+
110+
def test_invalid_parent(self):
111+
with util.create_modules('example') as mapping:
112+
finder = self.get_finder(mapping['.root'])
113+
example = finder.find_spec('example')
114+
with self.assertRaises(ValueError):
115+
list(finder.discover(example))
116+
117+
118+
(
119+
Frozen_TestFileFinder,
120+
Source_TestFileFinder,
121+
) = util.test_both(TestFileFinder, importlib=importlib, machinery=machinery)

‎Misc/NEWS.d/next/Library/2025-10-10-14-08-58.gh-issue-139899.09leRY.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Introduced :meth:`importlib.abc.MetaPathFinder.discover`
2+
and :meth:`importlib.abc.PathEntryFinder.discover` to allow module and submodule
3+
name discovery without assuming the use of traditional filesystem based imports.

0 commit comments

Comments
 (0)