~sschwarzer/ftputil

4f39a848e55ac15f5e17c72bdb052d8677bef788 — Stefan Schwarzer 9 years ago 3e5e618
Don't raise `PermanentError` for `host.path.isdir("/dir/subdir")`
if even the directory doesn't exist.

The fix was rather simple, but now ftputil does a lot more behind the
scenes; it traverses up from the path given as argument and does
a stat call on each of these traversals.

This means that in `mock_ftplib.MockSession`, we can no longer have
mixed directory strings for Unix and MS format, so some refactoring of
`MockSession` and changing of some dependent tests was necessary.
M ftputil/ftp_stat.py => ftputil/ftp_stat.py +6 -6
@@ 493,12 493,12 @@ class _Stat(object):
                  "can't stat remote root directory")
        dirname, basename = self._path.split(path)
        # If even the directory doesn't exist and we don't want the
        # exception, treat it the same as if the path wasn't found in
        # the directory's contents (compare below). The use of `isdir`
        # here causes a recursion but that should be ok because that
        # will at the latest stop when we've got to the root directory.
#         if not self._path.isdir(dirname) and not _exception_for_missing_path:
#             return None
        # exception, treat it the same as if the path wasn't found in the
        # directory's contents (compare below). The use of `isdir` here
        # causes a recursion but that should be ok because that will at
        # the latest stop when we've gotten to the root directory.
        if not self._path.isdir(dirname) and not _exception_for_missing_path:
            return None
        # Loop through all lines of the directory listing. We
        # probably won't need all lines for the particular path but
        # we want to collect as many stat results in the cache as

M test/mock_ftplib.py => test/mock_ftplib.py +27 -16
@@ 92,7 92,8 @@ drwxr-sr-x   2 45854    200           512 May  4  2000 sschwarzer
-rw-r--r--   1 45854    200          4605 Jan 19  1970 older
-rw-r--r--   1 45854    200          4605 Jan 19  2020 newer
lrwxrwxrwx   1 45854    200            21 Jan 19  2002 link -> sschwarzer/index.html
lrwxrwxrwx   1 45854    200            15 Jan 19  2002 bad_link -> python/bad_link""",
lrwxrwxrwx   1 45854    200            15 Jan 19  2002 bad_link -> python/bad_link
drwxr-sr-x   2 45854    200           512 May  4  2000 dir with spaces""",

      '/home/python': """\
lrwxrwxrwx   1 45854    200             7 Jan 19  2002 link_link -> ../link


@@ 117,21 118,6 @@ total 1
      # Fail when trying to write to this directory (the content isn't
      # relevant).
      'sschwarzer': "",

      '/home/msformat': """\
10-23-01  03:25PM       <DIR>          WindowsXP
12-07-01  02:05PM       <DIR>          XPLaunch
07-17-00  02:08PM             12266720 abcd.exe
07-17-00  02:08PM                89264 O2KKeys.exe""",

      '/home/msformat/XPLaunch': """\
10-23-01  03:25PM       <DIR>          WindowsXP
12-07-01  02:05PM       <DIR>          XPLaunch
12-07-01  02:05PM       <DIR>          empty
07-17-00  02:08PM             12266720 abcd.exe
07-17-00  02:08PM                89264 O2KKeys.exe""",

      '/home/msformat/XPLaunch/empty': "total 0",
    }

    # File content to be used (indirectly) with `transfercmd`.


@@ 224,3 210,28 @@ total 1
            self.closed = 1
            assert self._transfercmds == 0


class MockMSFormatSession(MockSession):

    dir_contents = {
      '/': """\
10-23-01  03:25PM       <DIR>          home""",

      '/home': """\
10-23-01  03:25PM       <DIR>          msformat""",

      '/home/msformat': """\
10-23-01  03:25PM       <DIR>          WindowsXP
12-07-01  02:05PM       <DIR>          XPLaunch
07-17-00  02:08PM             12266720 abcd.exe
07-17-00  02:08PM                89264 O2KKeys.exe""",

      '/home/msformat/XPLaunch': """\
10-23-01  03:25PM       <DIR>          WindowsXP
12-07-01  02:05PM       <DIR>          XPLaunch
12-07-01  02:05PM       <DIR>          empty
07-17-00  02:08PM             12266720 abcd.exe
07-17-00  02:08PM                89264 O2KKeys.exe""",

      '/home/msformat/XPLaunch/empty': "total 0",
    }

M test/test_ftp_path.py => test/test_ftp_path.py +5 -1
@@ 1,4 1,4 @@
# Copyright (C) 2003-2011, Stefan Schwarzer <sschwarzer@sschwarzer.net>
# Copyright (C) 2003-2012, Stefan Schwarzer <sschwarzer@sschwarzer.net>
# See the file LICENSE for licensing terms.

import ftplib


@@ 25,6 25,7 @@ class SessionWithInaccessibleLoginDirectory(mock_ftplib.MockSession):

class TestPath(unittest.TestCase):
    """Test operations in `FTPHost.path`."""

    def test_regular_isdir_isfile_islink(self):
        """Test regular `FTPHost._Path.isdir/isfile/islink`."""
        testdir = '/home/sschwarzer'


@@ 34,6 35,9 @@ class TestPath(unittest.TestCase):
        self.assertFalse(host.path.isdir('notthere'))
        self.assertFalse(host.path.isfile('notthere'))
        self.assertFalse(host.path.islink('notthere'))
        self.assertFalse(host.path.isdir('/notthere/notthere'))
        self.assertFalse(host.path.isfile('/notthere/notthere'))
        self.assertFalse(host.path.islink('/notthere/notthere'))
        # Test a directory
        self.assertTrue(host.path.isdir(testdir))
        self.assertFalse(host.path.isfile(testdir))

M test/test_ftp_stat.py => test/test_ftp_stat.py +18 -5
@@ 12,15 12,18 @@ from ftputil import ftp_error
from ftputil import ftp_stat

from test import test_base
from test import mock_ftplib


def test_stat():
    host = test_base.ftp_host_factory()
def test_stat(session_factory):
    host = test_base.ftp_host_factory(session_factory=session_factory)
    stat = ftp_stat._Stat(host)
    # Use Unix format parser explicitly.
    # Use Unix format parser explicitly. This doesn't exclude switching
    # to the MS format parser later if the test allows this switching.
    stat._parser = ftp_stat.UnixParser()
    return stat


def stat_tuple_to_seconds(t):
    """
    Return a float number representing the local time associated with


@@ 262,8 265,12 @@ class TestLstatAndStat(unittest.TestCase):
    Test `FTPHost.lstat` and `FTPHost.stat` (test currently only
    implemented for Unix server format).
    """

    def setUp(self):
        self.stat = test_stat()
        # Most tests in this class need the mock session class with
        # Unix format, so make this the default. Tests which need
        # the MS format, can overwrite `self.stat` later.
        self.stat = test_stat(session_factory=mock_ftplib.MockSession)

    def test_failing_lstat(self):
        """Test whether lstat fails for a nonexistent path."""


@@ 294,6 301,7 @@ class TestLstatAndStat(unittest.TestCase):

    def test_lstat_one_ms_file(self):
        """Test `lstat` for a file described in DOS-style format."""
        self.stat = test_stat(session_factory=mock_ftplib.MockMSFormatSession)
        stat_result = self.stat._lstat('/home/msformat/abcd.exe')
        self.assertEqual(stat_result._st_mtime_precision, 60)



@@ 319,6 327,7 @@ class TestLstatAndStat(unittest.TestCase):

    def test_lstat_one_ms_dir(self):
        """Test `lstat` for a directory described in DOS-style format."""
        self.stat = test_stat(session_factory=mock_ftplib.MockMSFormatSession)
        stat_result = self.stat._lstat('/home/msformat/WindowsXP')
        self.assertEqual(stat_result._st_mtime_precision, 60)



@@ 348,6 357,7 @@ class TestLstatAndStat(unittest.TestCase):
    #
    def test_parser_switching_with_permanent_error(self):
        """Test non-switching of parser format with `PermanentError`."""
        self.stat = test_stat(session_factory=mock_ftplib.MockMSFormatSession)
        self.assertEqual(self.stat._allow_parser_switching, True)
        # With these directory contents, we get a `ParserError` for
        # the Unix parser, so `_allow_parser_switching` can be


@@ 367,6 377,7 @@ class TestLstatAndStat(unittest.TestCase):

    def test_parser_switching_to_ms(self):
        """Test switching of parser from Unix to MS format."""
        self.stat = test_stat(session_factory=mock_ftplib.MockMSFormatSession)
        self.assertEqual(self.stat._allow_parser_switching, True)
        self.assertTrue(isinstance(self.stat._parser, ftp_stat.UnixParser))
        stat_result = self.stat._lstat("/home/msformat/abcd.exe")


@@ 377,6 388,7 @@ class TestLstatAndStat(unittest.TestCase):

    def test_parser_switching_regarding_empty_dir(self):
        """Test switching of parser if a directory is empty."""
        self.stat = test_stat(session_factory=mock_ftplib.MockMSFormatSession)
        self.assertEqual(self.stat._allow_parser_switching, True)
        result = self.stat._listdir("/home/msformat/XPLaunch/empty")
        self.assertEqual(result, [])


@@ 386,8 398,9 @@ class TestLstatAndStat(unittest.TestCase):

class TestListdir(unittest.TestCase):
    """Test `FTPHost.listdir`."""

    def setUp(self):
        self.stat = test_stat()
        self.stat = test_stat(session_factory=mock_ftplib.MockSession)

    def test_failing_listdir(self):
        """Test failing `FTPHost.listdir`."""

M test/test_ftputil.py => test/test_ftputil.py +23 -9
@@ 175,18 175,32 @@ class TestKeepAlive(unittest.TestCase):

class TestSetParser(unittest.TestCase):

    class TrivialParser(ftp_stat.Parser):
        """
        An instance of this parser always returns the same result
        from its `parse_line` method. This is all we need to check
        if ftputil uses the set parser. No actual parsing code is
        required here.
        """
        def __init__(self):
            # We can't use `os.stat("/home")` directly because we
            # later need the object's `_st_name` attribute, which
            # we can't set on a `os.stat` stat value.
            default_stat_result = ftp_stat.StatResult(os.stat("/home"))
            default_stat_result._st_name = "home"
            self.default_stat_result = default_stat_result

        def parse_line(self, line, time_shift=0.0):
            return self.default_stat_result

    def test_set_parser(self):
        """Test if the selected parser is used."""
        # This test isn't very practical but should help at least a bit ...
        host = test_base.ftp_host_factory()
        # Implicitly fix at Unix format
        files = host.listdir("/home/sschwarzer")
        self.assertEqual(files, ['chemeng', 'download', 'image', 'index.html',
          'os2', 'osup', 'publications', 'python', 'scios2'])
        host.set_parser(ftp_stat.MSParser())
        files = host.listdir("/home/msformat/XPLaunch")
        self.assertEqual(files, ['WindowsXP', 'XPLaunch', 'empty',
                                 'abcd.exe', 'O2KKeys.exe'])
        self.assertEqual(host._stat._allow_parser_switching, True)
        trivial_parser = TestSetParser.TrivialParser()
        host.set_parser(trivial_parser)
        stat_result = host.stat("/home")
        self.assertEqual(stat_result, trivial_parser.default_stat_result)
        self.assertEqual(host._stat._allow_parser_switching, False)



M test/test_real_ftp.py => test/test_real_ftp.py +9 -1
@@ 33,6 33,7 @@ def get_login_data():
    #return server, user, password
    return "localhost", 'ftptest', 'd605581757de5eb56d568a4419f4126e'


def utc_local_time_shift():
    """
    Return the expected time shift in seconds assuming the server


@@ 415,6 416,13 @@ class TestStat(RealFTPTest):
        calculated_time_shift = server_mtime - client_mtime
        self.assertFalse(abs(calculated_time_shift-host.time_shift()) > 120)

    def test_issomething_for_nonexistent_directory(self):
        host = self.host
        nonexistent_path = "/nonexistent/nonexistent"
        self.assertEqual(bool(host.path.isdir(nonexistent_path)), False)
        self.assertEqual(bool(host.path.isfile(nonexistent_path)), False)
        self.assertEqual(bool(host.path.islink(nonexistent_path)), False)

#    def test_special_broken_link(self):
#        # Test for ticket #39
#        # This test currently fails; I guess I'll postpone it until


@@ 839,5 847,5 @@ minutes because it has to wait to test the timezone calculation.
    server, user, password = get_login_data()
    unittest.main()
    import __main__
    #unittest.main(__main__, "TestOther.test_garbage_collection")
    #unittest.main(__main__, "TestStat.test_issomething_for_nonexistent_directory")