Skip to content

GH-110109: Move tests for pathlib.Path.walk() into main test classes. #110655

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Oct 28, 2023
Merged

GH-110109: Move tests for pathlib.Path.walk() into main test classes. #110655

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
17511b3
GH-110109: Move tests for `pathlib.Path.walk()` into main test classes.
barneygale Oct 10, 2023
47a81dd
Restore directory with no permissions.
barneygale Oct 10, 2023
9db41d0
Add check for path existence in `test_walk_symlink_location`
barneygale Oct 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
GH-110109: Move tests for pathlib.Path.walk() into main test classes. 
This adds some test coverage to `pathlib._PathBase.walk()`
  • Loading branch information
@barneygale
barneygale committed Oct 10, 2023
commit 17511b3c167039a67989b93d0270ecd9131ca338
347 changes: 162 additions & 185 deletions Lib/test/test_pathlib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1801,6 +1801,51 @@ def setUp(self):
p.joinpath('dirB', 'linkD').symlink_to(pathmod.join('..', 'dirB'))
p.joinpath('brokenLinkLoop').symlink_to('brokenLinkLoop')

def setUpWalk(self):
# Build:
# TESTFN/
# TEST1/ a file kid and two directory kids
# tmp1
# SUB1/ a file kid and a directory kid
# tmp2
# SUB11/ no kids
# SUB2/ a file kid and a dirsymlink kid
# tmp3
# link/ a symlink to TEST2
# broken_link
# broken_link2
# broken_link3
# TEST2/
# tmp4 a lone file
self.walk_path = self.cls(BASE, "TEST1")
self.sub1_path = self.walk_path / "SUB1"
self.sub11_path = self.sub1_path / "SUB11"
self.sub2_path = self.walk_path / "SUB2"
tmp1_path = self.walk_path / "tmp1"
tmp2_path = self.sub1_path / "tmp2"
tmp3_path = self.sub2_path / "tmp3"
self.link_path = self.sub2_path / "link"
t2_path = self.cls(BASE, "TEST2")
tmp4_path = self.cls(BASE, "TEST2", "tmp4")
broken_link_path = self.sub2_path / "broken_link"
broken_link2_path = self.sub2_path / "broken_link2"

self.sub11_path.mkdir(parents=True)
self.sub2_path.mkdir(parents=True)
t2_path.mkdir(parents=True)

for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path:
with path.open("w", encoding='utf-8') as f:
f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n")

if self.can_symlink:
self.link_path.symlink_to(t2_path)
broken_link_path.symlink_to('broken')
broken_link2_path.symlink_to(self.cls('tmp3', 'broken'))
self.sub2_tree = (self.sub2_path, [], ["broken_link", "broken_link2", "link", "tmp3"])
else:
self.sub2_tree = (self.sub2_path, [], ["tmp3"])

def tearDown(self):
cls = self.cls
cls._files.clear()
Expand Down Expand Up @@ -2614,6 +2659,119 @@ def test_complex_symlinks_relative(self):
def test_complex_symlinks_relative_dot_dot(self):
self._check_complex_symlinks(os.path.join('dirA', '..'))

def test_walk_topdown(self):
self.setUpWalk()
walker = self.walk_path.walk()
entry = next(walker)
entry[1].sort() # Ensure we visit SUB1 before SUB2
self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"]))
entry = next(walker)
self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"]))
entry = next(walker)
self.assertEqual(entry, (self.sub11_path, [], []))
entry = next(walker)
entry[1].sort()
entry[2].sort()
self.assertEqual(entry, self.sub2_tree)
with self.assertRaises(StopIteration):
next(walker)

def test_walk_prune(self):
self.setUpWalk()
# Prune the search.
all = []
for root, dirs, files in self.walk_path.walk():
all.append((root, dirs, files))
if 'SUB1' in dirs:
# Note that this also mutates the dirs we appended to all!
dirs.remove('SUB1')

self.assertEqual(len(all), 2)
self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"]))

all[1][-1].sort()
all[1][1].sort()
self.assertEqual(all[1], self.sub2_tree)

def test_walk_bottom_up(self):
self.setUpWalk()
seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False
for path, dirnames, filenames in self.walk_path.walk(top_down=False):
if path == self.walk_path:
self.assertFalse(seen_testfn)
self.assertTrue(seen_sub1)
self.assertTrue(seen_sub2)
self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"])
self.assertEqual(filenames, ["tmp1"])
seen_testfn = True
elif path == self.sub1_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub1)
self.assertTrue(seen_sub11)
self.assertEqual(dirnames, ["SUB11"])
self.assertEqual(filenames, ["tmp2"])
seen_sub1 = True
elif path == self.sub11_path:
self.assertFalse(seen_sub1)
self.assertFalse(seen_sub11)
self.assertEqual(dirnames, [])
self.assertEqual(filenames, [])
seen_sub11 = True
elif path == self.sub2_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub2)
self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1]))
self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2]))
seen_sub2 = True
else:
raise AssertionError(f"Unexpected path: {path}")
self.assertTrue(seen_testfn)

def test_walk_follow_symlinks(self):
if not self.can_symlink:
self.skipTest("symlinks required")
self.setUpWalk()
walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.link_path:
self.assertEqual(dirs, [])
self.assertEqual(files, ["tmp4"])
break
else:
self.fail("Didn't follow symlink with follow_symlinks=True")

def test_walk_symlink_location(self):
if not self.can_symlink:
self.skipTest("symlinks required")
self.setUpWalk()
# Tests whether symlinks end up in filenames or dirnames depending
# on the `follow_symlinks` argument.
walk_it = self.walk_path.walk(follow_symlinks=False)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", files)
break
else:
self.fail("symlink not found")

walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", dirs)
break

def test_walk_above_recursion_limit(self):
self.setUpWalk()
recursion_limit = 40
# directory_depth > recursion_limit
directory_depth = recursion_limit + 10
base = self.cls(BASE, 'deep')
path = self.cls(base, *(['d'] * directory_depth))
path.mkdir(parents=True)

with set_recursion_limit(recursion_limit):
list(base.walk())
list(base.walk(top_down=False))

class DummyPathWithSymlinks(DummyPath):
def readlink(self):
Expand Down Expand Up @@ -3187,178 +3345,8 @@ def test_passing_kwargs_deprecated(self):
with self.assertWarns(DeprecationWarning):
self.cls(foo="bar")


class WalkTests(unittest.TestCase):

def setUp(self):
self.addCleanup(os_helper.rmtree, os_helper.TESTFN)

# Build:
# TESTFN/
# TEST1/ a file kid and two directory kids
# tmp1
# SUB1/ a file kid and a directory kid
# tmp2
# SUB11/ no kids
# SUB2/ a file kid and a dirsymlink kid
# tmp3
# SUB21/ not readable
# tmp5
# link/ a symlink to TEST2
# broken_link
# broken_link2
# broken_link3
# TEST2/
# tmp4 a lone file
self.walk_path = pathlib.Path(os_helper.TESTFN, "TEST1")
self.sub1_path = self.walk_path / "SUB1"
self.sub11_path = self.sub1_path / "SUB11"
self.sub2_path = self.walk_path / "SUB2"
sub21_path= self.sub2_path / "SUB21"
tmp1_path = self.walk_path / "tmp1"
tmp2_path = self.sub1_path / "tmp2"
tmp3_path = self.sub2_path / "tmp3"
tmp5_path = sub21_path / "tmp3"
self.link_path = self.sub2_path / "link"
t2_path = pathlib.Path(os_helper.TESTFN, "TEST2")
tmp4_path = pathlib.Path(os_helper.TESTFN, "TEST2", "tmp4")
broken_link_path = self.sub2_path / "broken_link"
broken_link2_path = self.sub2_path / "broken_link2"
broken_link3_path = self.sub2_path / "broken_link3"

os.makedirs(self.sub11_path)
os.makedirs(self.sub2_path)
os.makedirs(sub21_path)
os.makedirs(t2_path)

for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path:
with open(path, "x", encoding='utf-8') as f:
f.write(f"I'm {path} and proud of it. Blame test_pathlib.\n")

if os_helper.can_symlink():
os.symlink(os.path.abspath(t2_path), self.link_path)
os.symlink('broken', broken_link_path, True)
os.symlink(pathlib.Path('tmp3', 'broken'), broken_link2_path, True)
os.symlink(pathlib.Path('SUB21', 'tmp5'), broken_link3_path, True)
self.sub2_tree = (self.sub2_path, ["SUB21"],
["broken_link", "broken_link2", "broken_link3",
"link", "tmp3"])
else:
self.sub2_tree = (self.sub2_path, ["SUB21"], ["tmp3"])

if not is_emscripten:
# Emscripten fails with inaccessible directories.
os.chmod(sub21_path, 0)
try:
os.listdir(sub21_path)
except PermissionError:
self.addCleanup(os.chmod, sub21_path, stat.S_IRWXU)
else:
os.chmod(sub21_path, stat.S_IRWXU)
os.unlink(tmp5_path)
os.rmdir(sub21_path)
del self.sub2_tree[1][:1]

def test_walk_topdown(self):
walker = self.walk_path.walk()
entry = next(walker)
entry[1].sort() # Ensure we visit SUB1 before SUB2
self.assertEqual(entry, (self.walk_path, ["SUB1", "SUB2"], ["tmp1"]))
entry = next(walker)
self.assertEqual(entry, (self.sub1_path, ["SUB11"], ["tmp2"]))
entry = next(walker)
self.assertEqual(entry, (self.sub11_path, [], []))
entry = next(walker)
entry[1].sort()
entry[2].sort()
self.assertEqual(entry, self.sub2_tree)
with self.assertRaises(StopIteration):
next(walker)

def test_walk_prune(self, walk_path=None):
if walk_path is None:
walk_path = self.walk_path
# Prune the search.
all = []
for root, dirs, files in walk_path.walk():
all.append((root, dirs, files))
if 'SUB1' in dirs:
# Note that this also mutates the dirs we appended to all!
dirs.remove('SUB1')

self.assertEqual(len(all), 2)
self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"]))

all[1][-1].sort()
all[1][1].sort()
self.assertEqual(all[1], self.sub2_tree)

def test_file_like_path(self):
self.test_walk_prune(FakePath(self.walk_path).__fspath__())

def test_walk_bottom_up(self):
seen_testfn = seen_sub1 = seen_sub11 = seen_sub2 = False
for path, dirnames, filenames in self.walk_path.walk(top_down=False):
if path == self.walk_path:
self.assertFalse(seen_testfn)
self.assertTrue(seen_sub1)
self.assertTrue(seen_sub2)
self.assertEqual(sorted(dirnames), ["SUB1", "SUB2"])
self.assertEqual(filenames, ["tmp1"])
seen_testfn = True
elif path == self.sub1_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub1)
self.assertTrue(seen_sub11)
self.assertEqual(dirnames, ["SUB11"])
self.assertEqual(filenames, ["tmp2"])
seen_sub1 = True
elif path == self.sub11_path:
self.assertFalse(seen_sub1)
self.assertFalse(seen_sub11)
self.assertEqual(dirnames, [])
self.assertEqual(filenames, [])
seen_sub11 = True
elif path == self.sub2_path:
self.assertFalse(seen_testfn)
self.assertFalse(seen_sub2)
self.assertEqual(sorted(dirnames), sorted(self.sub2_tree[1]))
self.assertEqual(sorted(filenames), sorted(self.sub2_tree[2]))
seen_sub2 = True
else:
raise AssertionError(f"Unexpected path: {path}")
self.assertTrue(seen_testfn)

@os_helper.skip_unless_symlink
def test_walk_follow_symlinks(self):
walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.link_path:
self.assertEqual(dirs, [])
self.assertEqual(files, ["tmp4"])
break
else:
self.fail("Didn't follow symlink with follow_symlinks=True")

@os_helper.skip_unless_symlink
def test_walk_symlink_location(self):
# Tests whether symlinks end up in filenames or dirnames depending
# on the `follow_symlinks` argument.
walk_it = self.walk_path.walk(follow_symlinks=False)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", files)
break
else:
self.fail("symlink not found")

walk_it = self.walk_path.walk(follow_symlinks=True)
for root, dirs, files in walk_it:
if root == self.sub2_path:
self.assertIn("link", dirs)
break

def test_walk_bad_dir(self):
self.setUpWalk()
errors = []
walk_it = self.walk_path.walk(on_error=errors.append)
root, dirs, files = next(walk_it)
Expand All @@ -3379,9 +3367,10 @@ def test_walk_bad_dir(self):
path1new.rename(path1)

def test_walk_many_open_files(self):
self.setUpWalk()
depth = 30
base = pathlib.Path(os_helper.TESTFN, 'deep')
path = pathlib.Path(base, *(['d']*depth))
base = self.cls(BASE, 'deep')
path = self.cls(base, *(['d']*depth))
path.mkdir(parents=True)

iters = [base.walk(top_down=False) for _ in range(100)]
Expand All @@ -3399,18 +3388,6 @@ def test_walk_many_open_files(self):
self.assertEqual(next(it), expected)
path = path / 'd'

def test_walk_above_recursion_limit(self):
recursion_limit = 40
# directory_depth > recursion_limit
directory_depth = recursion_limit + 10
base = pathlib.Path(os_helper.TESTFN, 'deep')
path = pathlib.Path(base, *(['d'] * directory_depth))
path.mkdir(parents=True)

with set_recursion_limit(recursion_limit):
list(base.walk())
list(base.walk(top_down=False))


@only_posix
class PosixPathTest(PathTest):
Expand Down