KaneAI - World’s First E2E Software Testing Agent.Book a Demo Join TestMu Conference

How to use os_walk method in molecule

Best Python code snippet using molecule_python

search.py

Source:search.py

1# -*- coding: utf-8 -*-2"""3Module with file search utilities.4"""5import os6import re7# -----------------------------------------------------------------------------8def find_files(root,9               prefix  = None,10               suffix  = None,11               dirname = None,12               direxcl = None):13    """14    Find files.15    ---16    type:   function17    ...18    """19    if direxcl is None:20        direxcl = [r'^\..*$']21    pathincl_regex = r'^.*'22    if dirname is not None:23        pathincl_regex += dirname + os.sep24    if prefix is not None:25        prefix = prefix.replace('.', r'\.')26        pathincl_regex += prefix + r'.*'27    if suffix is not None:28        suffix = suffix.replace('.', r'\.')29        pathincl_regex += suffix30    pathincl_regex += r'$'31    pathincl = [pathincl_regex]32    return filtered_filepath_generator(33        root     = root,34        direxcl  = direxcl,35        pathincl = pathincl,36        pathexcl = None)37# -----------------------------------------------------------------------------38def filtered_dirpath_generator(root,39                               direxcl  = None,40                               pathincl = None,41                               pathexcl = None):42    """43    Return generator of dirpaths from root, filtered using regex lists.44    ---45    type:   function46    ...47    """48    return _dirpath_from_os_walk_filter(49            os_walk  = _dirname_filtered_os_walk_gen(root, direxcl = direxcl),50            pathincl = pathincl,51            pathexcl = pathexcl)52# -----------------------------------------------------------------------------53def filtered_filepath_generator(root,54                                direxcl  = None,55                                pathincl = None,56                                pathexcl = None):57    """58    Return generator of filepaths from root, filtered using regex lists.59    ---60    type:   function61    ...62    """63    return _filepath_from_os_walk_filter(64            os_walk  = _dirname_filtered_os_walk_gen(root, direxcl = direxcl),65            pathincl = pathincl,66            pathexcl = pathexcl)67# -----------------------------------------------------------------------------68def _dirpath_from_os_walk_filter(os_walk,69                                 pathincl = None,70                                 pathexcl = None):71    """72    Return filter of dirpaths, adapted to take an iterable of os.walk tuples.73    ---74    type:   function75    ...76    """77    return _filepath_regex_filter(78        _adapt_os_walk_to_dirpath(os_walk), pathincl, pathexcl)79# -----------------------------------------------------------------------------80def _filepath_from_os_walk_filter(os_walk,81                                  pathincl = None,82                                  pathexcl = None):83    """84    Return filter of filepaths, adapted to take an iterable of os.walk tuples.85    ---86    type:   function87    ...88    """89    return _filepath_regex_filter(90        _adapt_os_walk_to_filepath(os_walk), pathincl, pathexcl)91# -----------------------------------------------------------------------------92def _adapt_os_walk_to_dirpath(os_walk):93    """94    Return adapter converting os.walk tuple iterable into dirpath iterable.95    Intended to process the output of os_walk and96    dirname_filter functions.97    ---98    type:   function99    ...100    """101    for (current_path, dir_list, _) in os_walk:102        for dir_name in dir_list:103            yield os.path.join(current_path, dir_name)104# -----------------------------------------------------------------------------105def _adapt_os_walk_to_filepath(os_walk):106    """107    Return adapter converting os.walk tuple iterable into filepath iterable.108    Intended to process the output of os_walk and109    dirname_filter functions.110    ---111    type:   function112    ...113    """114    for (current_path, _, file_list) in os_walk:115        for file_name in sorted(file_list):116            yield os.path.join(current_path, file_name)117# -----------------------------------------------------------------------------118def _dirname_filtered_os_walk_gen(root,119                                  direxcl     = None,120                                  onerror     = None,121                                  followlinks = False):122    """123    Return generator of os.walk tuples, filtered using regex lists.124    ---125    type:   function126    ...127    """128    return _dirname_regex_filter(129        os_walk = os.walk(root,130                          topdown     = True,131                          onerror     = onerror,132                          followlinks = followlinks),133        excl    = direxcl)134# -----------------------------------------------------------------------------135def _dirname_regex_filter(os_walk,136                          excl = None):137    """138    Filter tuples generated by os.walk. Recursion limited by directory name.139    The supplied indicator function is used to140    decide if a directory subtree should be141    recursed into or not.142    ---143    type:   function144    ...145    """146    dirname_indicator_func = get_dual_regex_indicator_fcn(excl = excl)147    return _dirname_filter(os_walk, dirname_indicator_func)148# -----------------------------------------------------------------------------149def _dirname_filter(os_walk, dirname_indicator_func):150    """151    Filter tuples generated by os.walk. Recursion limited by directory name.152    The supplied indicator function is used to153    decide if a directory subtree should be154    recursed into or not.155    ---156    type:   function157    ...158    """159    for (current_path, subdir_list, file_list) in os_walk:160        if not subdir_list:161            subdir_list[:] = []162        else:163            subdir_list[:] = sorted(164                path for path in subdir_list if dirname_indicator_func(path))165        yield (current_path, subdir_list, file_list)166# -----------------------------------------------------------------------------167def _filepath_regex_filter(iter_filepaths,168                           incl = None,169                           excl = None):170    """171    Filter for filepaths, filtering specified using regex lists.172    ---173    type:   function174    ...175    """176    filepath_indicator_func = get_dual_regex_indicator_fcn(incl, excl)177    return (path for path in iter_filepaths if filepath_indicator_func(path))178# -----------------------------------------------------------------------------179def get_dual_regex_indicator_fcn(incl=None, excl=None):180    """181    Indicator function for strings based on a pair of compiled regexes.182        - Returns True if incl matches and excl does not.183        - If incl is not specified or None, it always matches (always include).184        - If excl is not specified or None, it never matches (never exclude).185    ---186    type:   function187    ...188    """189    is_incl = _get_regex_indicator_fcn(incl, default = True)190    is_excl = _get_regex_indicator_fcn(excl, default = False)191    return lambda item: is_incl(item) and not is_excl(item)192# -----------------------------------------------------------------------------193def _get_regex_indicator_fcn(regex_list=None, default=False):194    """195    Return an indicator function defined by the specified regular expression.196    ---197    type:   function198    ...199    """200    if regex_list:201        regex = _compile_regex_list(regex_list)202        return lambda item: (regex.match(item) is not None)203    else:204        return lambda item: (default)205# -----------------------------------------------------------------------------206def _compile_regex_list(regex_list):207    """208    Compile a list of regex strings into a regular expression object.209    ---210    type:   function211    ...212    """213    combined = "(" + ")|(".join(regex_list) + ")"214    compiled = re.compile(combined)215    return compiled216# -----------------------------------------------------------------------------217def find_ancestor_dir_containing(dir_path, marker_name, allow_dir=True):218    """219    Search for an ancestor directory of dir_path that contains a marker.220    This function identifies the closest (deepest)221    ancestor directory of dir_path that contains a222    file or directory named as specified by the223    marker_name parameter.224    It works by visiting each ancestor directory225    in dir_path in turn, starting at dir_path and226    proceeding up towards the root of the file-227    system hierarchy. At each step, it checks to228    see if a file or directory with the specified229    name exists. If it finds this marker, it returns230    the containing directory. If not, it continues231    towards the root. If, after reaching the root,232    no marker has yet been found, a SearchError233    exception is raised.234    This function was created to help identify235    the root of the local working copy, given a236    path within it. It should also be possible237    to use this function to help identify the238    root of various other filesystem hierarchies,239    given the emplacement of suitable marker files240    or directories.241    ---242    type:   function243    ...244    """245    for dir_path in _walk_towards_root_generator(os.path.realpath(dir_path)):246        path_marker = os.path.join(dir_path, marker_name)247        is_file     = os.path.isfile(path_marker)248        if allow_dir:249            is_dir    = os.path.isdir(path_marker)250            is_marker = is_file or is_dir251        else:252            is_marker = is_file253        if is_marker:254            return dir_path255    raise RuntimeError("Could not find marker {name}".format(256                                                        name = marker_name))257# -----------------------------------------------------------------------------258def _walk_towards_root_generator(dir_path):259    """260    Iterate over ancestor directories from dir_path up to the filesystem root.261    This function generates a sequence of directory262    paths starting with dir_path and progressively263    returning the parent directory of each until264    the filesystem root directory is reached.265    ---266    type:   function267    ...268    """269    prev_path = None270    while dir_path != prev_path:271        yield dir_path272        prev_path = dir_path...

benchmark.py

Source:benchmark.py

...60                raise betterwalk.posix_error(path)61        return names62else:63    raise NotImplementedError64def os_walk(top, topdown=True, onerror=None, followlinks=False):65    """Identical to os.walk(), but use ctypes-based listdir() so benchmark66    against ctypes-based iterdir_stat() is valid.67    """68    try:69        names = os_listdir(top)70    except OSError as err:71        if onerror is not None:72            onerror(err)73        return74    dirs, nondirs = [], []75    for name in names:76        if os.path.isdir(os.path.join(top, name)):77            dirs.append(name)78        else:79            nondirs.append(name)80    if topdown:81        yield top, dirs, nondirs82    for name in dirs:83        new_path = os.path.join(top, name)84        if followlinks or not os.path.islink(new_path):85            for x in os_walk(new_path, topdown, onerror, followlinks):86                yield x87    if not topdown:88        yield top, dirs, nondirs89def create_tree(path, depth=DEPTH):90    """Create a directory tree at path with given depth, and NUM_DIRS and91    NUM_FILES at each level.92    """93    os.mkdir(path)94    for i in range(NUM_FILES):95        filename = os.path.join(path, 'file{0:03}.txt'.format(i))96        with open(filename, 'wb') as f:97            line = b'The quick brown fox jumps over the lazy dog.\n'98            if i == 0:99                # So we have at least one big file per directory100                f.write(line * 20000)101            else:102                f.write(line * i * 10)103    if depth <= 1:104        return105    for i in range(NUM_DIRS):106        dirname = os.path.join(path, 'dir{0:03}'.format(i))107        create_tree(dirname, depth - 1)108def get_tree_size(path):109    """Return total size of all files in directory tree at path."""110    size = 0111    try:112        for name, st in betterwalk.iterdir_stat(path, fields=['st_mode_type', 'st_size']):113            if stat.S_ISDIR(st.st_mode):114                size += get_tree_size(os.path.join(path, name))115            else:116                size += st.st_size117    except OSError:118        pass119    return size120def benchmark(path, get_size=False):121    sizes = {}122    if get_size:123        def do_os_walk():124            size = 0125            for root, dirs, files in os_walk(path):126                for filename in files:127                    fullname = os.path.join(root, filename)128                    size += os.path.getsize(fullname)129            sizes['os_walk'] = size130        def do_betterwalk():131            sizes['betterwalk'] = get_tree_size(path)132    else:133        def do_os_walk():134            for root, dirs, files in os_walk(path):135                pass136        def do_betterwalk():137            for root, dirs, files in betterwalk.walk(path):138                pass139    # Run this once first to cache things, so we're not benchmarking I/O140    print("Priming the system's cache...")141    do_betterwalk()142    # Use the best of 3 time for each of them to eliminate high outliers143    os_walk_time = 1000000144    betterwalk_time = 1000000145    N = 3146    for i in range(N):147        print('Benchmarking walks on {0}, repeat {1}/{2}...'.format(148            path, i + 1, N))...

test_filemanager.py

Source:test_filemanager.py

1""" Test the Filemanager module """2import os3import time4import pyres.filemanager5from mock import patch6class TestFilemanager(object):7    """ test the open functionality """8    def test_create(self, tmpdir):9        """  just call the cstor for the object """10        assert self11        filemgr = pyres.filemanager.FileManager(base_dir=tmpdir.dirname)12        assert filemgr.does_filesystem_exist()13    @patch('shutil.copyfile')14    def test_copy(self, mock_copy):   # pylint: disable=W062115        """  copy files correctly """16        assert self17        episodes = [18            pyres.episode.Episode(19                date=time.localtime(),20                title='title',21                url='url',22                podcast='podcast',23                size=1234,24                state=1,25                # one of these two required26                base_path='base_path',27                file_name='file_name'28            ),29            pyres.episode.Episode(30                date=time.gmtime(),  # use gm time to get a different value31                title='title',32                url='url',33                podcast='podcast',34                size=1234,35                state=1,36                # one of these two required37                base_path='base_path',38                file_name='file_name2'39            )40        ]41        copy_base = 'copy_base'42        filemgr = pyres.filemanager.FileManager(base_dir=copy_base)43        assert filemgr44        filemgr.copy_episodes_to_player(episodes)45        assert mock_copy.called46        assert mock_copy.call_count == 247    @patch('shutil.copyfile')48    @patch('pyres.filemanager.utils.mkdir_p')49    @patch('pyres.filemanager.os.walk')50    def test_audiobook_copy(self, os_walk, mkdir_p,51                            copyfile):   # pylint: disable=W062152        """  test simple file copy """53        assert self54        assert mkdir_p  # to appease pylint55        os_walk.return_value = [56            ('/foo', ['bar', ], ('baz',)),57            ('/foo/bar', [], ('spam', 'eggs')),58        ]59        copy_base = 'copy_base'60        filemgr = pyres.filemanager.FileManager(base_dir=copy_base)61        filemgr.copy_audiobook("source")62        assert copyfile.called63        print(copyfile.call_args_list)64        assert copyfile.call_count == 365        assert copyfile.any_call('/foo/baz')66        assert copyfile.any_call('/foo/bar/spam')67        assert copyfile.any_call('/foo/bar/eggs')68    @patch('shutil.copyfile')69    @patch('pyres.filemanager.utils.mkdir_p')70    @patch('pyres.filemanager.os.walk')71    def test_audiobook_name(self, os_walk, mkdir_p,72                            copyfile):   # pylint: disable=W062173        """  test that names with numbers are converted correctly """74        assert self75        assert mkdir_p  # to appease pylint76        def single_file(filename, expected):77            """ Test a single file """78            os_walk.return_value = [79                # if os.walk returns a root with leading /, os.path.join80                # doesn't add the filemanager base_dir to the path.  This81                # makes it a bit easier to write these tests82                ('/foo', [], (filename,)),83            ]84            filemgr.copy_audiobook("source")85            copyfile.assert_called_with(os.path.join('/foo', filename),86                                        os.path.join('/foo', expected))87        copy_base = 'copy_base'88        filemgr = pyres.filemanager.FileManager(base_dir=copy_base)89        assert filemgr90        # one digit91        single_file('1one', '01one')92        single_file('o1ne', 'o01ne')93        single_file('one1', 'one01')94        # two digits95        single_file('11two', '11two')96        single_file('t11wo', 't11wo')97        single_file('two11', 'two11')98        # three digits99        single_file('123three', '123three')100        single_file('th123ree', 'th123ree')101        single_file('three123', 'three123')102        # mp3 extension not changed103        single_file('1baz.mp3', '01baz.mp3')104        single_file('b1az.mp3', 'b01az.mp3')105        single_file('baz1.mp3', 'baz01.mp3')106        # test multiple substitutions107        single_file('1b1a1z.1mp3', '01b01a01z.01mp3')108    @patch('shutil.copyfile')109    @patch('pyres.filemanager.utils.mkdir_p')110    @patch('pyres.filemanager.os.walk')111    def test_audiobook_dest_dir(self, os_walk, mkdir_p,112                                copyfile):   # pylint: disable=W0621113        """  test that the dest dir parameters is used correctly """114        assert self115        assert mkdir_p  # to appease pylint116        copy_base = 'copy_base'117        filemgr = pyres.filemanager.FileManager(base_dir=copy_base)118        assert filemgr119        os_walk.return_value = [120            ('source', [], ("wilma",)),121        ]122        filemgr.copy_audiobook("source", "dest")123        print(copyfile.call_args_list)124        copyfile.assert_called_with(os.path.join('source', 'wilma'),125                                    os.path.join('copy_base', 'dest', 'source',126                                                 'wilma'))127    @patch('shutil.copyfile')128    @patch('pyres.filemanager.utils.mkdir_p')129    @patch('pyres.filemanager.os.walk')130    def test_audiobook_deep_dir(self, os_walk, mkdir_p,131                                copyfile):   # pylint: disable=W0621132        """  test that directory structures are preserved """133        assert self134        assert mkdir_p  # to appease pylint135        copy_base = 'copy_base'136        filemgr = pyres.filemanager.FileManager(base_dir=copy_base)137        assert filemgr138        os_walk.return_value = [139            # see note above about leading slashes on top level directory140            ('/first', ['second', ], ("wilma",)),141            ('/first/second', ['third', ], ("wilma",)),142            ('/first/second/third', ['', ], ("wilma", "fred")),143        ]144        filemgr.copy_audiobook("source")145        copyfile.assert_any_call(os.path.join('/first', 'wilma'),146                                 os.path.join('/first', 'wilma'))147        copyfile.assert_any_call(os.path.join('/first/second', 'wilma'),148                                 os.path.join('/first/second', 'wilma'))149        copyfile.assert_any_call(os.path.join('/first/second/third', 'wilma'),150                                 os.path.join('/first/second/third', 'wilma'))151        copyfile.assert_any_call(os.path.join('/first/second/third', 'fred'),...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.