Tachyon /Users/simon/.local/share/uv/python/cpython-3.15.0-macos-aarch64-none/lib/python3.15/shutil.py
1
Self Samples
27
Cumulative
2
Lines Hit
0.09%
% of Total
1
Max Self
26
Max Total
Intensity:
Cold Hot
Self Time
Total Time
Show All
Hot Only
Heat
Specialization
Line
Self
Total
Code
1
"""Utility functions for copying and archiving files and directory trees.
2
3
XXX The functions here don't copy the resource fork or other metadata on Mac.
4
5
"""
6
7
import os
8
import sys
9
import stat
10
import fnmatch
11
import collections
12
import errno
13
14
try:
15
import zlib
16
del zlib
17
_ZLIB_SUPPORTED = True
18
except ImportError:
19
_ZLIB_SUPPORTED = False
20
21
try:
22
import bz2
23
del bz2
24
_BZ2_SUPPORTED = True
25
except ImportError:
26
_BZ2_SUPPORTED = False
27
28
try:
29
import lzma
30
del lzma
31
_LZMA_SUPPORTED = True
32
except ImportError:
33
_LZMA_SUPPORTED = False
34
35
try:
36
from compression import zstd
37
del zstd
38
_ZSTD_SUPPORTED = True
39
except ImportError:
40
_ZSTD_SUPPORTED = False
41
42
_WINDOWS = os.name == 'nt'
43
posix = nt = None
44
if os.name == 'posix':
45
import posix
46
elif _WINDOWS:
47
import nt
48
49
if sys.platform == 'win32':
50
import _winapi
51
else:
52
_winapi = None
53
54
COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 256 * 1024
55
# This should never be removed, see rationale in:
56
# https://bugs.python.org/issue43743#msg393429
57
_USE_CP_SENDFILE = (hasattr(os, "sendfile")
58
and sys.platform.startswith(("linux", "android", "sunos")))
59
_USE_CP_COPY_FILE_RANGE = hasattr(os, "copy_file_range")
60
_HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS
61
62
# CMD defaults in Windows 10
63
_WIN_DEFAULT_PATHEXT = ".COM;.EXE;.BAT;.CMD;.VBS;.JS;.WS;.MSC"
64
65
__all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
66
"copytree", "move", "rmtree", "Error", "SpecialFileError",
67
"make_archive", "get_archive_formats",
68
"register_archive_format", "unregister_archive_format",
69
"get_unpack_formats", "register_unpack_format",
70
"unregister_unpack_format", "unpack_archive",
71
"ignore_patterns", "chown", "which", "get_terminal_size",
72
"SameFileError"]
73
# disk_usage is added later, if available on the platform
74
75
class Error(OSError):
76
pass
77
78
class SameFileError(Error):
79
"""Raised when source and destination are the same file."""
80
81
class SpecialFileError(OSError):
82
"""Raised when trying to do a kind of operation (e.g. copying) which is
83
not supported on a special file (e.g. a named pipe)"""
84
85
86
class ReadError(OSError):
87
"""Raised when an archive cannot be read"""
88
89
class RegistryError(Exception):
90
"""Raised when a registry operation with the archiving
91
and unpacking registries fails"""
92
93
class _GiveupOnFastCopy(Exception):
94
"""Raised as a signal to fallback on using raw read()/write()
95
file copy when fast-copy functions fail to do so.
96
"""
97
98
def _fastcopy_fcopyfile(fsrc, fdst, flags):
99
"""Copy a regular file content or metadata by using high-performance
100
fcopyfile(3) syscall (macOS).
101
"""
102
try:
103
infd = fsrc.fileno()
104
outfd = fdst.fileno()
105
except Exception as err:
106
raise _GiveupOnFastCopy(err) # not a regular file
107
108
try:
109
posix._fcopyfile(infd, outfd, flags)
110
except OSError as err:
111
err.filename = fsrc.name
112
err.filename2 = fdst.name
113
if err.errno in {errno.EINVAL, errno.ENOTSUP}:
114
raise _GiveupOnFastCopy(err)
115
else:
116
raise err from None
117
118
def _determine_linux_fastcopy_blocksize(infd):
119
"""Determine blocksize for fastcopying on Linux.
120
121
Hopefully the whole file will be copied in a single call.
122
The copying itself should be performed in a loop 'till EOF is
123
reached (0 return) so a blocksize smaller or bigger than the actual
124
file size should not make any difference, also in case the file
125
content changes while being copied.
126
"""
127
try:
128
blocksize = max(os.fstat(infd).st_size, 2 ** 23) # min 8 MiB
129
except OSError:
130
blocksize = 2 ** 27 # 128 MiB
131
# On 32-bit architectures truncate to 1 GiB to avoid OverflowError,
132
# see gh-82500.
133
if sys.maxsize < 2 ** 32:
134
blocksize = min(blocksize, 2 ** 30)
135
return blocksize
136
137
def _fastcopy_copy_file_range(fsrc, fdst):
138
"""Copy data from one regular mmap-like fd to another by using
139
a high-performance copy_file_range(2) syscall that gives filesystems
140
an opportunity to implement the use of reflinks or server-side copy.
141
142
This should work on Linux >= 4.5 only.
143
"""
144
try:
145
infd = fsrc.fileno()
146
outfd = fdst.fileno()
147
except Exception as err:
148
raise _GiveupOnFastCopy(err) # not a regular file
149
150
blocksize = _determine_linux_fastcopy_blocksize(infd)
151
offset = 0
152
while True:
153
try:
154
n_copied = os.copy_file_range(infd, outfd, blocksize, offset_dst=offset)
155
except OSError as err:
156
# ...in oder to have a more informative exception.
157
err.filename = fsrc.name
158
err.filename2 = fdst.name
159
160
if err.errno == errno.ENOSPC: # filesystem is full
161
raise err from None
162
163
# Give up on first call and if no data was copied.
164
if offset == 0 and os.lseek(outfd, 0, os.SEEK_CUR) == 0:
165
raise _GiveupOnFastCopy(err)
166
167
raise err
168
else:
169
if n_copied == 0:
170
# If no bytes have been copied yet, copy_file_range
171
# might silently fail.
172
# https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0
173
if offset == 0:
174
raise _GiveupOnFastCopy()
175
break
176
offset += n_copied
177
178
def _fastcopy_sendfile(fsrc, fdst):
179
"""Copy data from one regular mmap-like fd to another by using
180
high-performance sendfile(2) syscall.
181
This should work on Linux >= 2.6.33, Android and Solaris.
182
"""
183
# Note: copyfileobj() is left alone in order to not introduce any
184
# unexpected breakage. Possible risks by using zero-copy calls
185
# in copyfileobj() are:
186
# - fdst cannot be open in "a"(ppend) mode
187
# - fsrc and fdst may be open in "t"(ext) mode
188
# - fsrc may be a BufferedReader (which hides unread data in a buffer),
189
# GzipFile (which decompresses data), HTTPResponse (which decodes
190
# chunks).
191
# - possibly others (e.g. encrypted fs/partition?)
192
global _USE_CP_SENDFILE
193
try:
194
infd = fsrc.fileno()
195
outfd = fdst.fileno()
196
except Exception as err:
197
raise _GiveupOnFastCopy(err) # not a regular file
198
199
blocksize = _determine_linux_fastcopy_blocksize(infd)
200
offset = 0
201
while True:
202
try:
203
sent = os.sendfile(outfd, infd, offset, blocksize)
204
except OSError as err:
205
# ...in order to have a more informative exception.
206
err.filename = fsrc.name
207
err.filename2 = fdst.name
208
209
if err.errno == errno.ENOTSOCK:
210
# sendfile() on this platform (probably Linux < 2.6.33)
211
# does not support copies between regular files (only
212
# sockets).
213
_USE_CP_SENDFILE = False
214
raise _GiveupOnFastCopy(err)
215
216
if err.errno == errno.ENOSPC: # filesystem is full
217
raise err from None
218
219
# Give up on first call and if no data was copied.
220
if offset == 0 and os.lseek(outfd, 0, os.SEEK_CUR) == 0:
221
raise _GiveupOnFastCopy(err)
222
223
raise err
224
else:
225
if sent == 0:
226
break # EOF
227
offset += sent
228
229
def _copyfileobj_readinto(fsrc, fdst, length=COPY_BUFSIZE):
230
"""readinto()/memoryview() based variant of copyfileobj().
231
*fsrc* must support readinto() method and both files must be
232
open in binary mode.
233
"""
234
# Localize variable access to minimize overhead.
235
fsrc_readinto = fsrc.readinto
236
fdst_write = fdst.write
237
with memoryview(bytearray(length)) as mv:
238
while True:
239
n = fsrc_readinto(mv)
240
if not n:
241
break
242
elif n < length:
243
with mv[:n] as smv:
244
fdst_write(smv)
245
break
246
else:
247
fdst_write(mv)
248
249
def copyfileobj(fsrc, fdst, length=0):
250
"""copy data from file-like object fsrc to file-like object fdst"""
251
if not length:
252
length = COPY_BUFSIZE
253
# Localize variable access to minimize overhead.
254
fsrc_read = fsrc.read
255
fdst_write = fdst.write
256
1
while buf := fsrc_read(length):
257
1
26
fdst_write(buf)
258
259
def _samefile(src, dst):
260
# Macintosh, Unix.
261
if isinstance(src, os.DirEntry) and hasattr(os.path, 'samestat'):
262
try:
263
return os.path.samestat(src.stat(), os.stat(dst))
264
except OSError:
265
return False
266
267
if hasattr(os.path, 'samefile'):
268
try:
269
return os.path.samefile(src, dst)
270
except OSError:
271
return False
272
273
# All other platforms: check for same pathname.
274
return (os.path.normcase(os.path.abspath(src)) ==
275
os.path.normcase(os.path.abspath(dst)))
276
277
def _stat(fn):
278
return fn.stat() if isinstance(fn, os.DirEntry) else os.stat(fn)
279
280
def _islink(fn):
281
return fn.is_symlink() if isinstance(fn, os.DirEntry) else os.path.islink(fn)
282
283
def copyfile(src, dst, *, follow_symlinks=True):
284
"""Copy data from src to dst in the most efficient way possible.
285
286
If follow_symlinks is not set and src is a symbolic link, a new
287
symlink will be created instead of copying the file it points to.
288
289
"""
290
sys.audit("shutil.copyfile", src, dst)
291
292
if _samefile(src, dst):
293
raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
294
295
file_size = 0
296
for i, fn in enumerate([src, dst]):
297
try:
298
st = _stat(fn)
299
except OSError:
300
# File most likely does not exist
301
pass
302
else:
303
# XXX What about other special files? (sockets, devices...)
304
if stat.S_ISFIFO(st.st_mode):
305
fn = fn.path if isinstance(fn, os.DirEntry) else fn
306
raise SpecialFileError("`%s` is a named pipe" % fn)
307
if _WINDOWS and i == 0:
308
file_size = st.st_size
309
310
if not follow_symlinks and _islink(src):
311
os.symlink(os.readlink(src), dst)
312
else:
313
with open(src, 'rb') as fsrc:
314
try:
315
with open(dst, 'wb') as fdst:
316
# macOS
317
if _HAS_FCOPYFILE:
318
try:
319
_fastcopy_fcopyfile(fsrc, fdst, posix._COPYFILE_DATA)
320
return dst
321
except _GiveupOnFastCopy:
322
pass
323
# Linux / Android / Solaris
324
elif _USE_CP_SENDFILE or _USE_CP_COPY_FILE_RANGE:
325
# reflink may be implicit in copy_file_range.
326
if _USE_CP_COPY_FILE_RANGE:
327
try:
328
_fastcopy_copy_file_range(fsrc, fdst)
329
return dst
330
except _GiveupOnFastCopy:
331
pass
332
if _USE_CP_SENDFILE:
333
try:
334
_fastcopy_sendfile(fsrc, fdst)
335
return dst
336
except _GiveupOnFastCopy:
337
pass
338
# Windows, see:
339
# https://github.com/python/cpython/pull/7160#discussion_r195405230
340
elif _WINDOWS and file_size > 0:
341
_copyfileobj_readinto(fsrc, fdst, min(file_size, COPY_BUFSIZE))
342
return dst
343
344
copyfileobj(fsrc, fdst)
345
346
# Issue 43219, raise a less confusing exception
347
except IsADirectoryError as e:
348
if not os.path.exists(dst):
349
raise FileNotFoundError(f'Directory does not exist: {dst}') from e
350
else:
351
raise
352
353
return dst
354
355
def copymode(src, dst, *, follow_symlinks=True):
356
"""Copy mode bits from src to dst.
357
358
If follow_symlinks is not set, symlinks aren't followed if and only
359
if both `src` and `dst` are symlinks. If `lchmod` isn't available
360
(e.g. Linux) this method does nothing.
361
362
"""
363
sys.audit("shutil.copymode", src, dst)
364
365
if not follow_symlinks and _islink(src) and os.path.islink(dst):
366
if hasattr(os, 'lchmod'):
367
stat_func, chmod_func = os.lstat, os.lchmod
368
else:
369
return
370
else:
371
stat_func = _stat
372
if os.name == 'nt' and os.path.islink(dst):
373
def chmod_func(*args):
374
os.chmod(*args, follow_symlinks=True)
375
else:
376
chmod_func = os.chmod
377
378
st = stat_func(src)
379
chmod_func(dst, stat.S_IMODE(st.st_mode))
380
381
if hasattr(os, 'listxattr'):
382
def _copyxattr(src, dst, *, follow_symlinks=True):
383
"""Copy extended filesystem attributes from `src` to `dst`.
384
385
Overwrite existing attributes.
386
387
If `follow_symlinks` is false, symlinks won't be followed.
388
389
"""
390
391
try:
392
names = os.listxattr(src, follow_symlinks=follow_symlinks)
393
except OSError as e:
394
if e.errno not in (errno.ENOTSUP, errno.ENODATA, errno.EINVAL):
395
raise
396
return
397
for name in names:
398
try:
399
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
400
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
401
except OSError as e:
402
if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA,
403
errno.EINVAL, errno.EACCES):
404
raise
405
else:
406
def _copyxattr(*args, **kwargs):
407
pass
408
409
def copystat(src, dst, *, follow_symlinks=True):
410
"""Copy file metadata
411
412
Copy the permission bits, last access time, last modification time, and
413
flags from `src` to `dst`. On Linux, copystat() also copies the "extended
414
attributes" where possible. The file contents, owner, and group are
415
unaffected. `src` and `dst` are path-like objects or path names given as
416
strings.
417
418
If the optional flag `follow_symlinks` is not set, symlinks aren't
419
followed if and only if both `src` and `dst` are symlinks.
420
"""
421
sys.audit("shutil.copystat", src, dst)
422
423
def _nop(*args, ns=None, follow_symlinks=None):
424
pass
425
426
# follow symlinks (aka don't not follow symlinks)
427
follow = follow_symlinks or not (_islink(src) and os.path.islink(dst))
428
if follow:
429
# use the real function if it exists
430
def lookup(name):
431
return getattr(os, name, _nop)
432
else:
433
# use the real function only if it exists
434
# *and* it supports follow_symlinks
435
def lookup(name):
436
fn = getattr(os, name, _nop)
437
if fn in os.supports_follow_symlinks:
438
return fn
439
return _nop
440
441
if isinstance(src, os.DirEntry):
442
st = src.stat(follow_symlinks=follow)
443
else:
444
st = lookup("stat")(src, follow_symlinks=follow)
445
mode = stat.S_IMODE(st.st_mode)
446
lookup("utime")(dst, ns=(st.st_atime_ns, st.st_mtime_ns),
447
follow_symlinks=follow)
448
# We must copy extended attributes before the file is (potentially)
449
# chmod()'ed read-only, otherwise setxattr() will error with -EACCES.
450
_copyxattr(src, dst, follow_symlinks=follow)
451
try:
452
lookup("chmod")(dst, mode, follow_symlinks=follow)
453
except NotImplementedError:
454
# if we got a NotImplementedError, it's because
455
# * follow_symlinks=False,
456
# * lchown() is unavailable, and
457
# * either
458
# * fchownat() is unavailable or
459
# * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW.
460
# (it returned ENOSUP.)
461
# therefore we're out of options--we simply cannot chown the
462
# symlink. give up, suppress the error.
463
# (which is what shutil always did in this circumstance.)
464
pass
465
if hasattr(st, 'st_flags'):
466
try:
467
lookup("chflags")(dst, st.st_flags, follow_symlinks=follow)
468
except OSError as why:
469
for err in 'EOPNOTSUPP', 'ENOTSUP':
470
if hasattr(errno, err) and why.errno == getattr(errno, err):
471
break
472
else:
473
raise
474
475
def copy(src, dst, *, follow_symlinks=True):
476
"""Copy data and mode bits ("cp src dst"). Return the file's destination.
477
478
The destination may be a directory.
479
480
If follow_symlinks is false, symlinks won't be followed. This
481
resembles GNU's "cp -P src dst".
482
483
If source and destination are the same file, a SameFileError will be
484
raised.
485
486
"""
487
if os.path.isdir(dst):
488
dst = os.path.join(dst, os.path.basename(src))
489
copyfile(src, dst, follow_symlinks=follow_symlinks)
490
copymode(src, dst, follow_symlinks=follow_symlinks)
491
return dst
492
493
def copy2(src, dst, *, follow_symlinks=True):
494
"""Copy data and metadata. Return the file's destination.
495
496
Metadata is copied with copystat(). Please see the copystat function
497
for more information.
498
499
The destination may be a directory.
500
501
If follow_symlinks is false, symlinks won't be followed. This
502
resembles GNU's "cp -P src dst".
503
"""
504
if os.path.isdir(dst):
505
dst = os.path.join(dst, os.path.basename(src))
506
507
if hasattr(_winapi, "CopyFile2"):
508
src_ = os.fsdecode(src)
509
dst_ = os.fsdecode(dst)
510
flags = _winapi.COPY_FILE_ALLOW_DECRYPTED_DESTINATION # for compat
511
if not follow_symlinks:
512
flags |= _winapi.COPY_FILE_COPY_SYMLINK
513
try:
514
_winapi.CopyFile2(src_, dst_, flags)
515
return dst
516
except OSError as exc:
517
if (exc.winerror == _winapi.ERROR_PRIVILEGE_NOT_HELD
518
and not follow_symlinks):
519
# Likely encountered a symlink we aren't allowed to create.
520
# Fall back on the old code
521
pass
522
elif exc.winerror == _winapi.ERROR_ACCESS_DENIED:
523
# Possibly encountered a hidden or readonly file we can't
524
# overwrite. Fall back on old code
525
pass
526
else:
527
raise
528
529
copyfile(src, dst, follow_symlinks=follow_symlinks)
530
copystat(src, dst, follow_symlinks=follow_symlinks)
531
return dst
532
533
def ignore_patterns(*patterns):
534
"""Function that can be used as copytree() ignore parameter.
535
536
Patterns is a sequence of glob-style patterns
537
that are used to exclude files"""
538
def _ignore_patterns(path, names):
539
ignored_names = []
540
for pattern in patterns:
541
ignored_names.extend(fnmatch.filter(names, pattern))
542
return set(ignored_names)
543
return _ignore_patterns
544
545
def _copytree(entries, src, dst, symlinks, ignore, copy_function,
546
ignore_dangling_symlinks, dirs_exist_ok=False):
547
if ignore is not None:
548
ignored_names = ignore(os.fspath(src), [x.name for x in entries])
549
else:
550
ignored_names = ()
551
552
os.makedirs(dst, exist_ok=dirs_exist_ok)
553
errors = []
554
use_srcentry = copy_function is copy2 or copy_function is copy
555
556
for srcentry in entries:
557
if srcentry.name in ignored_names:
558
continue
559
srcname = os.path.join(src, srcentry.name)
560
dstname = os.path.join(dst, srcentry.name)
561
srcobj = srcentry if use_srcentry else srcname
562
try:
563
is_symlink = srcentry.is_symlink()
564
if is_symlink and os.name == 'nt':
565
# Special check for directory junctions, which appear as
566
# symlinks but we want to recurse.
567
lstat = srcentry.stat(follow_symlinks=False)
568
if lstat.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT:
569
is_symlink = False
570
if is_symlink:
571
linkto = os.readlink(srcname)
572
if symlinks:
573
# We can't just leave it to `copy_function` because legacy
574
# code with a custom `copy_function` may rely on copytree
575
# doing the right thing.
576
os.symlink(linkto, dstname)
577
copystat(srcobj, dstname, follow_symlinks=not symlinks)
578
else:
579
# ignore dangling symlink if the flag is on
580
if not os.path.exists(linkto) and ignore_dangling_symlinks:
581
continue
582
# otherwise let the copy occur. copy2 will raise an error
583
if srcentry.is_dir():
584
copytree(srcobj, dstname, symlinks, ignore,
585
copy_function, ignore_dangling_symlinks,
586
dirs_exist_ok)
587
else:
588
copy_function(srcobj, dstname)
589
elif srcentry.is_dir():
590
copytree(srcobj, dstname, symlinks, ignore, copy_function,
591
ignore_dangling_symlinks, dirs_exist_ok)
592
else:
593
# Will raise a SpecialFileError for unsupported file types
594
copy_function(srcobj, dstname)
595
# catch the Error from the recursive copytree so that we can
596
# continue with other files
597
except Error as err:
598
errors.extend(err.args[0])
599
except OSError as why:
600
errors.append((srcname, dstname, str(why)))
601
try:
602
copystat(src, dst)
603
except OSError as why:
604
# Copying file access times may fail on Windows
605
if getattr(why, 'winerror', None) is None:
606
errors.append((src, dst, str(why)))
607
if errors:
608
raise Error(errors)
609
return dst
610
611
def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2,
612
ignore_dangling_symlinks=False, dirs_exist_ok=False):
613
"""Recursively copy a directory tree and return the destination directory.
614
615
If exception(s) occur, an Error is raised with a list of reasons.
616
617
If the optional symlinks flag is true, symbolic links in the
618
source tree result in symbolic links in the destination tree; if
619
it is false, the contents of the files pointed to by symbolic
620
links are copied. If the file pointed to by the symlink doesn't
621
exist, an exception will be added in the list of errors raised in
622
an Error exception at the end of the copy process.
623
624
You can set the optional ignore_dangling_symlinks flag to true if you
625
want to silence this exception. Notice that this has no effect on
626
platforms that don't support os.symlink.
627
628
The optional ignore argument is a callable. If given, it
629
is called with the `src` parameter, which is the directory
630
being visited by copytree(), and `names` which is the list of
631
`src` contents, as returned by os.listdir():
632
633
callable(src, names) -> ignored_names
634
635
Since copytree() is called recursively, the callable will be
636
called once for each directory that is copied. It returns a
637
list of names relative to the `src` directory that should
638
not be copied.
639
640
The optional copy_function argument is a callable that will be used
641
to copy each file. It will be called with the source path and the
642
destination path as arguments. By default, copy2() is used, but any
643
function that supports the same signature (like copy()) can be used.
644
645
If dirs_exist_ok is false (the default) and `dst` already exists, a
646
`FileExistsError` is raised. If `dirs_exist_ok` is true, the copying
647
operation will continue if it encounters existing directories, and files
648
within the `dst` tree will be overwritten by corresponding files from the
649
`src` tree.
650
"""
651
sys.audit("shutil.copytree", src, dst)
652
with os.scandir(src) as itr:
653
entries = list(itr)
654
return _copytree(entries=entries, src=src, dst=dst, symlinks=symlinks,
655
ignore=ignore, copy_function=copy_function,
656
ignore_dangling_symlinks=ignore_dangling_symlinks,
657
dirs_exist_ok=dirs_exist_ok)
658
659
if hasattr(os.stat_result, 'st_file_attributes'):
660
def _rmtree_islink(st):
661
return (stat.S_ISLNK(st.st_mode) or
662
(st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT
663
and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT))
664
else:
665
def _rmtree_islink(st):
666
return stat.S_ISLNK(st.st_mode)
667
668
# version vulnerable to race conditions
669
def _rmtree_unsafe(path, dir_fd, onexc):
670
if dir_fd is not None:
671
raise NotImplementedError("dir_fd unavailable on this platform")
672
try:
673
st = os.lstat(path)
674
except OSError as err:
675
onexc(os.lstat, path, err)
676
return
677
try:
678
if _rmtree_islink(st):
679
# symlinks to directories are forbidden, see bug #1669
680
raise OSError("Cannot call rmtree on a symbolic link")
681
except OSError as err:
682
onexc(os.path.islink, path, err)
683
# can't continue even if onexc hook returns
684
return
685
def onerror(err):
686
if not isinstance(err, FileNotFoundError):
687
onexc(os.scandir, err.filename, err)
688
results = os.walk(path, topdown=False, onerror=onerror, followlinks=os._walk_symlinks_as_files)
689
for dirpath, dirnames, filenames in results:
690
for name in dirnames:
691
fullname = os.path.join(dirpath, name)
692
try:
693
os.rmdir(fullname)
694
except FileNotFoundError:
695
continue
696
except OSError as err:
697
onexc(os.rmdir, fullname, err)
698
for name in filenames:
699
fullname = os.path.join(dirpath, name)
700
try:
701
os.unlink(fullname)
702
except FileNotFoundError:
703
continue
704
except OSError as err:
705
onexc(os.unlink, fullname, err)
706
try:
707
os.rmdir(path)
708
except FileNotFoundError:
709
pass
710
except OSError as err:
711
onexc(os.rmdir, path, err)
712
713
# Version using fd-based APIs to protect against races
714
def _rmtree_safe_fd(path, dir_fd, onexc):
715
# While the unsafe rmtree works fine on bytes, the fd based does not.
716
if isinstance(path, bytes):
717
path = os.fsdecode(path)
718
stack = [(os.lstat, dir_fd, path, None)]
719
try:
720
while stack:
721
_rmtree_safe_fd_step(stack, onexc)
722
finally:
723
# Close any file descriptors still on the stack.
724
while stack:
725
func, fd, path, entry = stack.pop()
726
if func is not os.close:
727
continue
728
try:
729
os.close(fd)
730
except OSError as err:
731
onexc(os.close, path, err)
732
733
def _rmtree_safe_fd_step(stack, onexc):
734
# Each stack item has four elements:
735
# * func: The first operation to perform: os.lstat, os.close or os.rmdir.
736
# Walking a directory starts with an os.lstat() to detect symlinks; in
737
# this case, func is updated before subsequent operations and passed to
738
# onexc() if an error occurs.
739
# * dirfd: Open file descriptor, or None if we're processing the top-level
740
# directory given to rmtree() and the user didn't supply dir_fd.
741
# * path: Path of file to operate upon. This is passed to onexc() if an
742
# error occurs.
743
# * orig_entry: os.DirEntry, or None if we're processing the top-level
744
# directory given to rmtree(). We used the cached stat() of the entry to
745
# save a call to os.lstat() when walking subdirectories.
746
func, dirfd, path, orig_entry = stack.pop()
747
name = path if orig_entry is None else orig_entry.name
748
parent_fd = None if func is os.close else dirfd
749
try:
750
if func is os.close:
751
os.close(dirfd)
752
return
753
if func is os.rmdir:
754
os.rmdir(name, dir_fd=dirfd)
755
return
756
757
# Note: To guard against symlink races, we use the standard
758
# lstat()/open()/fstat() trick.
759
assert func is os.lstat
760
if orig_entry is None:
761
orig_st = os.lstat(name, dir_fd=dirfd)
762
else:
763
orig_st = orig_entry.stat(follow_symlinks=False)
764
765
func = os.open # For error reporting.
766
topfd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dirfd)
767
768
func = os.path.islink # For error reporting.
769
try:
770
if not os.path.samestat(orig_st, os.fstat(topfd)):
771
# Symlinks to directories are forbidden, see GH-46010.
772
raise OSError("Cannot call rmtree on a symbolic link")
773
stack.append((os.rmdir, dirfd, path, orig_entry))
774
finally:
775
stack.append((os.close, topfd, path, orig_entry))
776
777
func = os.scandir # For error reporting.
778
with os.scandir(topfd) as scandir_it:
779
entries = list(scandir_it)
780
for entry in entries:
781
fullname = os.path.join(path, entry.name)
782
try:
783
if entry.is_dir(follow_symlinks=False):
784
# Traverse into sub-directory.
785
stack.append((os.lstat, topfd, fullname, entry))
786
continue
787
except FileNotFoundError:
788
continue
789
except OSError:
790
pass
791
try:
792
os.unlink(entry.name, dir_fd=topfd)
793
except FileNotFoundError:
794
continue
795
except OSError as err:
796
onexc(os.unlink, fullname, err, direntry=entry, dir_fd=topfd)
797
except FileNotFoundError as err:
798
if orig_entry is None or func is os.close:
799
err.filename = path
800
onexc(func, path, err, direntry=orig_entry, dir_fd=parent_fd)
801
except OSError as err:
802
err.filename = path
803
onexc(func, path, err, direntry=orig_entry, dir_fd=parent_fd)
804
805
_use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <=
806
os.supports_dir_fd and
807
os.scandir in os.supports_fd and
808
os.stat in os.supports_follow_symlinks)
809
_rmtree_impl = _rmtree_safe_fd if _use_fd_functions else _rmtree_unsafe
810
811
def rmtree(path, ignore_errors=False, onerror=None, *, onexc=None, dir_fd=None,
812
_onexc_kwargs=False):
813
"""Recursively delete a directory tree.
814
815
If dir_fd is not None, it should be a file descriptor open to a directory;
816
path will then be relative to that directory.
817
dir_fd may not be implemented on your platform.
818
If it is unavailable, using it will raise a NotImplementedError.
819
820
If ignore_errors is set, errors are ignored; otherwise, if onexc or
821
onerror is set, it is called to handle the error with arguments (func,
822
path, exc_info) where func is platform and implementation dependent;
823
path is the argument to that function that caused it to fail; and
824
the value of exc_info describes the exception. For onexc it is the
825
exception instance, and for onerror it is a tuple as returned by
826
sys.exc_info(). If ignore_errors is false and both onexc and
827
onerror are None, the exception is reraised.
828
829
onerror is deprecated and only remains for backwards compatibility.
830
If both onerror and onexc are set, onerror is ignored and onexc is used.
831
"""
832
833
sys.audit("shutil.rmtree", path, dir_fd)
834
if ignore_errors:
835
def onexc(*args, **kwargs):
836
pass
837
elif onerror is None and onexc is None:
838
def onexc(*args, **kwargs):
839
raise
840
elif onexc is None:
841
if onerror is None:
842
def onexc(*args, **kwargs):
843
raise
844
else:
845
# delegate to onerror
846
def onexc(*args, **kwargs):
847
func, path, exc = args
848
if exc is None:
849
exc_info = None, None, None
850
else:
851
exc_info = type(exc), exc, exc.__traceback__
852
return onerror(func, path, exc_info)
853
elif not _onexc_kwargs:
854
# Only the internal caller in tempfile asks for the extra arguments.
855
_onexc = onexc
856
def onexc(func, path, err, **kwargs):
857
return _onexc(func, path, err)
858
859
_rmtree_impl(path, dir_fd, onexc)
860
861
# Allow introspection of whether or not the hardening against symlink
862
# attacks is supported on the current platform
863
rmtree.avoids_symlink_attacks = _use_fd_functions
864
865
def _basename(path):
866
"""A basename() variant which first strips the trailing slash, if present.
867
Thus we always get the last component of the path, even for directories.
868
869
path: Union[PathLike, str]
870
871
e.g.
872
>>> os.path.basename('/bar/foo')
873
'foo'
874
>>> os.path.basename('/bar/foo/')
875
''
876
>>> _basename('/bar/foo/')
877
'foo'
878
"""
879
path = os.fspath(path)
880
sep = os.path.sep + (os.path.altsep or '')
881
return os.path.basename(path.rstrip(sep))
882
883
def move(src, dst, copy_function=copy2):
884
"""Recursively move a file or directory to another location. This is
885
similar to the Unix "mv" command. Return the file or directory's
886
destination.
887
888
If dst is an existing directory or a symlink to a directory, then src is
889
moved inside that directory. The destination path in that directory must
890
not already exist.
891
892
If dst already exists but is not a directory, it may be overwritten
893
depending on os.rename() semantics.
894
895
os.rename() is preferably used if the source and destination are on the
896
same filesystem. In case os.rename() fails due to OSError (e.g. the user
897
has write permission to *dst* file but not to its parent directory),
898
this method falls back to using *copy_function* silently.
899
Symlinks are also recreated under the new name if os.rename() fails
900
because of cross filesystem renames.
901
902
It's recommended to use os.rename() if atomic move is strictly required.
903
904
The optional `copy_function` argument is a callable that will be used
905
to copy the source or it will be delegated to `copytree`.
906
By default, copy2() is used, but any function that supports the same
907
signature (like copy()) can be used.
908
909
A lot more could be done here... A look at a mv.c shows a lot of
910
the issues this implementation glosses over.
911
912
"""
913
sys.audit("shutil.move", src, dst)
914
real_dst = dst
915
if os.path.isdir(dst):
916
if _samefile(src, dst) and not os.path.islink(src):
917
# We might be on a case insensitive filesystem,
918
# perform the rename anyway.
919
os.rename(src, dst)
920
return
921
922
# Using _basename instead of os.path.basename is important, as we must
923
# ignore any trailing slash to avoid the basename returning ''
924
real_dst = os.path.join(dst, _basename(src))
925
926
if os.path.exists(real_dst):
927
raise Error("Destination path '%s' already exists" % real_dst)
928
try:
929
os.rename(src, real_dst)
930
except OSError:
931
if os.path.islink(src):
932
linkto = os.readlink(src)
933
os.symlink(linkto, real_dst)
934
os.unlink(src)
935
elif os.path.isdir(src):
936
if _destinsrc(src, dst):
937
raise Error("Cannot move a directory '%s' into itself"
938
" '%s'." % (src, dst))
939
if (_is_immutable(src)
940
or (not os.access(src, os.W_OK) and os.listdir(src)
941
and sys.platform == 'darwin')):
942
raise PermissionError("Cannot move the non-empty directory "
943
"'%s': Lacking write permission to '%s'."
944
% (src, src))
945
copytree(src, real_dst, copy_function=copy_function,
946
symlinks=True)
947
rmtree(src)
948
else:
949
copy_function(src, real_dst)
950
os.unlink(src)
951
return real_dst
952
953
def _destinsrc(src, dst):
954
src = os.path.realpath(src)
955
dst = os.path.realpath(dst)
956
if not src.endswith(os.path.sep):
957
src += os.path.sep
958
if not dst.endswith(os.path.sep):
959
dst += os.path.sep
960
return dst.startswith(src)
961
962
def _is_immutable(src):
963
st = _stat(src)
964
immutable_states = [stat.UF_IMMUTABLE, stat.SF_IMMUTABLE]
965
return hasattr(st, 'st_flags') and st.st_flags in immutable_states
966
967
def _get_gid(name):
968
"""Returns a gid, given a group name."""
969
if name is None:
970
return None
971
972
try:
973
from grp import getgrnam
974
except ImportError:
975
return None
976
977
try:
978
result = getgrnam(name)
979
except KeyError:
980
result = None
981
if result is not None:
982
return result[2]
983
return None
984
985
def _get_uid(name):
986
"""Returns an uid, given a user name."""
987
if name is None:
988
return None
989
990
try:
991
from pwd import getpwnam
992
except ImportError:
993
return None
994
995
try:
996
result = getpwnam(name)
997
except KeyError:
998
result = None
999
if result is not None:
1000
return result[2]
1001
return None
1002
1003
def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0,
1004
owner=None, group=None, logger=None, root_dir=None):
1005
"""Create a (possibly compressed) tar file from all the files under
1006
'base_dir'.
1007
1008
'compress' must be "gzip" (the default), "bzip2", "xz", "zst", or None.
1009
1010
'owner' and 'group' can be used to define an owner and a group for the
1011
archive that is being built. If not provided, the current owner and group
1012
will be used.
1013
1014
The output tar file will be named 'base_name' + ".tar", possibly plus
1015
the appropriate compression extension (".gz", ".bz2", ".xz", or ".zst").
1016
1017
Returns the output filename.
1018
"""
1019
if compress is None:
1020
tar_compression = ''
1021
elif _ZLIB_SUPPORTED and compress == 'gzip':
1022
tar_compression = 'gz'
1023
elif _BZ2_SUPPORTED and compress == 'bzip2':
1024
tar_compression = 'bz2'
1025
elif _LZMA_SUPPORTED and compress == 'xz':
1026
tar_compression = 'xz'
1027
elif _ZSTD_SUPPORTED and compress == 'zst':
1028
tar_compression = 'zst'
1029
else:
1030
raise ValueError("bad value for 'compress', or compression format not "
1031
"supported : {0}".format(compress))
1032
1033
import tarfile # late import for breaking circular dependency
1034
1035
compress_ext = '.' + tar_compression if compress else ''
1036
archive_name = base_name + '.tar' + compress_ext
1037
archive_dir = os.path.dirname(archive_name)
1038
1039
if archive_dir and not os.path.exists(archive_dir):
1040
if logger is not None:
1041
logger.info("creating %s", archive_dir)
1042
if not dry_run:
1043
os.makedirs(archive_dir)
1044
1045
# creating the tarball
1046
if logger is not None:
1047
logger.info('Creating tar archive')
1048
1049
uid = _get_uid(owner)
1050
gid = _get_gid(group)
1051
1052
def _set_uid_gid(tarinfo):
1053
if gid is not None:
1054
tarinfo.gid = gid
1055
tarinfo.gname = group
1056
if uid is not None:
1057
tarinfo.uid = uid
1058
tarinfo.uname = owner
1059
return tarinfo
1060
1061
if not dry_run:
1062
tar = tarfile.open(archive_name, 'w|%s' % tar_compression)
1063
arcname = base_dir
1064
if root_dir is not None:
1065
base_dir = os.path.join(root_dir, base_dir)
1066
try:
1067
tar.add(base_dir, arcname, filter=_set_uid_gid)
1068
finally:
1069
tar.close()
1070
1071
if root_dir is not None:
1072
archive_name = os.path.abspath(archive_name)
1073
return archive_name
1074
1075
def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0,
1076
logger=None, owner=None, group=None, root_dir=None):
1077
"""Create a zip file from all the files under 'base_dir'.
1078
1079
The output zip file will be named 'base_name' + ".zip". Returns the
1080
name of the output zip file.
1081
"""
1082
import zipfile # late import for breaking circular dependency
1083
1084
zip_filename = base_name + ".zip"
1085
archive_dir = os.path.dirname(base_name)
1086
1087
if archive_dir and not os.path.exists(archive_dir):
1088
if logger is not None:
1089
logger.info("creating %s", archive_dir)
1090
if not dry_run:
1091
os.makedirs(archive_dir)
1092
1093
if logger is not None:
1094
logger.info("creating '%s' and adding '%s' to it",
1095
zip_filename, base_dir)
1096
1097
if not dry_run:
1098
with zipfile.ZipFile(zip_filename, "w",
1099
compression=zipfile.ZIP_DEFLATED) as zf:
1100
arcname = os.path.normpath(base_dir)
1101
if root_dir is not None:
1102
base_dir = os.path.join(root_dir, base_dir)
1103
base_dir = os.path.normpath(base_dir)
1104
if arcname != os.curdir:
1105
zf.write(base_dir, arcname)
1106
if logger is not None:
1107
logger.info("adding '%s'", base_dir)
1108
for dirpath, dirnames, filenames in os.walk(base_dir):
1109
arcdirpath = dirpath
1110
if root_dir is not None:
1111
arcdirpath = os.path.relpath(arcdirpath, root_dir)
1112
arcdirpath = os.path.normpath(arcdirpath)
1113
for name in sorted(dirnames):
1114
path = os.path.join(dirpath, name)
1115
arcname = os.path.join(arcdirpath, name)
1116
zf.write(path, arcname)
1117
if logger is not None:
1118
logger.info("adding '%s'", path)
1119
for name in filenames:
1120
path = os.path.join(dirpath, name)
1121
path = os.path.normpath(path)
1122
if os.path.isfile(path):
1123
arcname = os.path.join(arcdirpath, name)
1124
zf.write(path, arcname)
1125
if logger is not None:
1126
logger.info("adding '%s'", path)
1127
1128
if root_dir is not None:
1129
zip_filename = os.path.abspath(zip_filename)
1130
return zip_filename
1131
1132
_make_tarball.supports_root_dir = True
1133
_make_zipfile.supports_root_dir = True
1134
1135
# Maps the name of the archive format to a tuple containing:
1136
# * the archiving function
1137
# * extra keyword arguments
1138
# * description
1139
_ARCHIVE_FORMATS = {
1140
'tar': (_make_tarball, [('compress', None)],
1141
"uncompressed tar file"),
1142
}
1143
1144
if _ZLIB_SUPPORTED:
1145
_ARCHIVE_FORMATS['gztar'] = (_make_tarball, [('compress', 'gzip')],
1146
"gzip'ed tar-file")
1147
_ARCHIVE_FORMATS['zip'] = (_make_zipfile, [], "ZIP file")
1148
1149
if _BZ2_SUPPORTED:
1150
_ARCHIVE_FORMATS['bztar'] = (_make_tarball, [('compress', 'bzip2')],
1151
"bzip2'ed tar-file")
1152
1153
if _LZMA_SUPPORTED:
1154
_ARCHIVE_FORMATS['xztar'] = (_make_tarball, [('compress', 'xz')],
1155
"xz'ed tar-file")
1156
1157
if _ZSTD_SUPPORTED:
1158
_ARCHIVE_FORMATS['zstdtar'] = (_make_tarball, [('compress', 'zst')],
1159
"zstd'ed tar-file")
1160
1161
def get_archive_formats():
1162
"""Returns a list of supported formats for archiving and unarchiving.
1163
1164
Each element of the returned sequence is a tuple (name, description)
1165
"""
1166
formats = [(name, registry[2]) for name, registry in
1167
_ARCHIVE_FORMATS.items()]
1168
formats.sort()
1169
return formats
1170
1171
def register_archive_format(name, function, extra_args=None, description=''):
1172
"""Registers an archive format.
1173
1174
name is the name of the format. function is the callable that will be
1175
used to create archives. If provided, extra_args is a sequence of
1176
(name, value) tuples that will be passed as arguments to the callable.
1177
description can be provided to describe the format, and will be returned
1178
by the get_archive_formats() function.
1179
"""
1180
if extra_args is None:
1181
extra_args = []
1182
if not callable(function):
1183
raise TypeError('The %s object is not callable' % function)
1184
if not isinstance(extra_args, (tuple, list)):
1185
raise TypeError('extra_args needs to be a sequence')
1186
for element in extra_args:
1187
if not isinstance(element, (tuple, list)) or len(element) !=2:
1188
raise TypeError('extra_args elements are : (arg_name, value)')
1189
1190
_ARCHIVE_FORMATS[name] = (function, extra_args, description)
1191
1192
def unregister_archive_format(name):
1193
del _ARCHIVE_FORMATS[name]
1194
1195
def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
1196
dry_run=0, owner=None, group=None, logger=None):
1197
"""Create an archive file (eg. zip or tar).
1198
1199
'base_name' is the name of the file to create, minus any format-specific
1200
extension; 'format' is the archive format: one of "zip", "tar", "gztar",
1201
"bztar", "xztar", or "zstdtar". Or any other registered format.
1202
1203
'root_dir' is a directory that will be the root directory of the
1204
archive; ie. we typically chdir into 'root_dir' before creating the
1205
archive. 'base_dir' is the directory where we start archiving from;
1206
ie. 'base_dir' will be the common prefix of all files and
1207
directories in the archive. 'root_dir' and 'base_dir' both default
1208
to the current directory. Returns the name of the archive file.
1209
1210
'owner' and 'group' are used when creating a tar archive. By default,
1211
uses the current owner and group.
1212
"""
1213
sys.audit("shutil.make_archive", base_name, format, root_dir, base_dir)
1214
try:
1215
format_info = _ARCHIVE_FORMATS[format]
1216
except KeyError:
1217
raise ValueError("unknown archive format '%s'" % format) from None
1218
1219
kwargs = {'dry_run': dry_run, 'logger': logger,
1220
'owner': owner, 'group': group}
1221
1222
func = format_info[0]
1223
for arg, val in format_info[1]:
1224
kwargs[arg] = val
1225
1226
base_name = os.fspath(base_name)
1227
1228
if base_dir is None:
1229
base_dir = os.curdir
1230
else:
1231
base_dir = os.fspath(base_dir)
1232
1233
supports_root_dir = getattr(func, 'supports_root_dir', False)
1234
save_cwd = None
1235
if root_dir is not None:
1236
root_dir = os.fspath(root_dir)
1237
stmd = os.stat(root_dir).st_mode
1238
if not stat.S_ISDIR(stmd):
1239
raise NotADirectoryError(errno.ENOTDIR, 'Not a directory', root_dir)
1240
1241
if supports_root_dir:
1242
kwargs['root_dir'] = root_dir
1243
else:
1244
save_cwd = os.getcwd()
1245
if logger is not None:
1246
logger.debug("changing into '%s'", root_dir)
1247
base_name = os.path.abspath(base_name)
1248
if not dry_run:
1249
os.chdir(root_dir)
1250
1251
try:
1252
filename = func(base_name, base_dir, **kwargs)
1253
finally:
1254
if save_cwd is not None:
1255
if logger is not None:
1256
logger.debug("changing back to '%s'", save_cwd)
1257
os.chdir(save_cwd)
1258
1259
return filename
1260
1261
1262
def get_unpack_formats():
1263
"""Returns a list of supported formats for unpacking.
1264
1265
Each element of the returned sequence is a tuple
1266
(name, extensions, description)
1267
"""
1268
formats = [(name, info[0], info[3]) for name, info in
1269
_UNPACK_FORMATS.items()]
1270
formats.sort()
1271
return formats
1272
1273
def _check_unpack_options(extensions, function, extra_args):
1274
"""Checks what gets registered as an unpacker."""
1275
# first make sure no other unpacker is registered for this extension
1276
existing_extensions = {}
1277
for name, info in _UNPACK_FORMATS.items():
1278
for ext in info[0]:
1279
existing_extensions[ext] = name
1280
1281
for extension in extensions:
1282
if extension in existing_extensions:
1283
msg = '%s is already registered for "%s"'
1284
raise RegistryError(msg % (extension,
1285
existing_extensions[extension]))
1286
1287
if not callable(function):
1288
raise TypeError('The registered function must be a callable')
1289
1290
1291
def register_unpack_format(name, extensions, function, extra_args=None,
1292
description=''):
1293
"""Registers an unpack format.
1294
1295
`name` is the name of the format. `extensions` is a list of extensions
1296
corresponding to the format.
1297
1298
`function` is the callable that will be
1299
used to unpack archives. The callable will receive archives to unpack.
1300
If it's unable to handle an archive, it needs to raise a ReadError
1301
exception.
1302
1303
If provided, `extra_args` is a sequence of
1304
(name, value) tuples that will be passed as arguments to the callable.
1305
description can be provided to describe the format, and will be returned
1306
by the get_unpack_formats() function.
1307
"""
1308
if extra_args is None:
1309
extra_args = []
1310
_check_unpack_options(extensions, function, extra_args)
1311
_UNPACK_FORMATS[name] = extensions, function, extra_args, description
1312
1313
def unregister_unpack_format(name):
1314
"""Removes the pack format from the registry."""
1315
del _UNPACK_FORMATS[name]
1316
1317
def _ensure_directory(path):
1318
"""Ensure that the parent directory of `path` exists"""
1319
dirname = os.path.dirname(path)
1320
if not os.path.isdir(dirname):
1321
os.makedirs(dirname)
1322
1323
def _unpack_zipfile(filename, extract_dir):
1324
"""Unpack zip `filename` to `extract_dir`
1325
"""
1326
import zipfile # late import for breaking circular dependency
1327
1328
if not zipfile.is_zipfile(filename):
1329
raise ReadError("%s is not a zip file" % filename)
1330
1331
with zipfile.ZipFile(filename) as zip:
1332
zip._ignore_invalid_names = True
1333
zip.extractall(extract_dir)
1334
1335
def _unpack_tarfile(filename, extract_dir, *, filter=None):
1336
"""Unpack tar/tar.gz/tar.bz2/tar.xz/tar.zst `filename` to `extract_dir`
1337
"""
1338
import tarfile # late import for breaking circular dependency
1339
try:
1340
tarobj = tarfile.open(filename)
1341
except tarfile.TarError:
1342
raise ReadError(
1343
"%s is not a compressed or uncompressed tar file" % filename)
1344
try:
1345
tarobj.extractall(extract_dir, filter=filter)
1346
finally:
1347
tarobj.close()
1348
1349
# Maps the name of the unpack format to a tuple containing:
1350
# * extensions
1351
# * the unpacking function
1352
# * extra keyword arguments
1353
# * description
1354
_UNPACK_FORMATS = {
1355
'tar': (['.tar'], _unpack_tarfile, [], "uncompressed tar file"),
1356
'zip': (['.zip'], _unpack_zipfile, [], "ZIP file"),
1357
}
1358
1359
if _ZLIB_SUPPORTED:
1360
_UNPACK_FORMATS['gztar'] = (['.tar.gz', '.tgz'], _unpack_tarfile, [],
1361
"gzip'ed tar-file")
1362
1363
if _BZ2_SUPPORTED:
1364
_UNPACK_FORMATS['bztar'] = (['.tar.bz2', '.tbz2'], _unpack_tarfile, [],
1365
"bzip2'ed tar-file")
1366
1367
if _LZMA_SUPPORTED:
1368
_UNPACK_FORMATS['xztar'] = (['.tar.xz', '.txz'], _unpack_tarfile, [],
1369
"xz'ed tar-file")
1370
1371
if _ZSTD_SUPPORTED:
1372
_UNPACK_FORMATS['zstdtar'] = (['.tar.zst', '.tzst'], _unpack_tarfile, [],
1373
"zstd'ed tar-file")
1374
1375
def _find_unpack_format(filename):
1376
for name, info in _UNPACK_FORMATS.items():
1377
for extension in info[0]:
1378
if filename.endswith(extension):
1379
return name
1380
return None
1381
1382
def unpack_archive(filename, extract_dir=None, format=None, *, filter=None):
1383
"""Unpack an archive.
1384
1385
`filename` is the name of the archive.
1386
1387
`extract_dir` is the name of the target directory, where the archive
1388
is unpacked. If not provided, the current working directory is used.
1389
1390
`format` is the archive format: one of "zip", "tar", "gztar", "bztar",
1391
"xztar", or "zstdtar". Or any other registered format. If not provided,
1392
unpack_archive will use the filename extension and see if an unpacker
1393
was registered for that extension.
1394
1395
In case none is found, a ValueError is raised.
1396
1397
If `filter` is given, it is passed to the underlying
1398
extraction function.
1399
"""
1400
sys.audit("shutil.unpack_archive", filename, extract_dir, format)
1401
1402
if extract_dir is None:
1403
extract_dir = os.getcwd()
1404
1405
extract_dir = os.fspath(extract_dir)
1406
filename = os.fspath(filename)
1407
1408
if filter is None:
1409
filter_kwargs = {}
1410
else:
1411
filter_kwargs = {'filter': filter}
1412
if format is not None:
1413
try:
1414
format_info = _UNPACK_FORMATS[format]
1415
except KeyError:
1416
raise ValueError("Unknown unpack format '{0}'".format(format)) from None
1417
1418
func = format_info[1]
1419
func(filename, extract_dir, **dict(format_info[2]), **filter_kwargs)
1420
else:
1421
# we need to look at the registered unpackers supported extensions
1422
format = _find_unpack_format(filename)
1423
if format is None:
1424
raise ReadError("Unknown archive format '{0}'".format(filename))
1425
1426
func = _UNPACK_FORMATS[format][1]
1427
kwargs = dict(_UNPACK_FORMATS[format][2]) | filter_kwargs
1428
func(filename, extract_dir, **kwargs)
1429
1430
1431
if hasattr(os, 'statvfs'):
1432
1433
__all__.append('disk_usage')
1434
_ntuple_diskusage = collections.namedtuple('usage', 'total used free')
1435
_ntuple_diskusage.total.__doc__ = 'Total space in bytes'
1436
_ntuple_diskusage.used.__doc__ = 'Used space in bytes'
1437
_ntuple_diskusage.free.__doc__ = 'Free space in bytes'
1438
1439
def disk_usage(path):
1440
"""Return disk usage statistics about the given path.
1441
1442
Returned value is a named tuple with attributes 'total', 'used' and
1443
'free', which are the amount of total, used and free space, in bytes.
1444
"""
1445
st = os.statvfs(path)
1446
free = st.f_bavail * st.f_frsize
1447
total = st.f_blocks * st.f_frsize
1448
used = (st.f_blocks - st.f_bfree) * st.f_frsize
1449
return _ntuple_diskusage(total, used, free)
1450
1451
elif _WINDOWS:
1452
1453
__all__.append('disk_usage')
1454
_ntuple_diskusage = collections.namedtuple('usage', 'total used free')
1455
1456
def disk_usage(path):
1457
"""Return disk usage statistics about the given path.
1458
1459
Returned values is a named tuple with attributes 'total', 'used' and
1460
'free', which are the amount of total, used and free space, in bytes.
1461
"""
1462
total, free = nt._getdiskusage(path)
1463
used = total - free
1464
return _ntuple_diskusage(total, used, free)
1465
1466
1467
def chown(path, user=None, group=None, *, dir_fd=None, follow_symlinks=True):
1468
"""Change owner user and group of the given path.
1469
1470
user and group can be the uid/gid or the user/group names, and in that case,
1471
they are converted to their respective uid/gid.
1472
1473
If dir_fd is set, it should be an open file descriptor to the directory to
1474
be used as the root of *path* if it is relative.
1475
1476
If follow_symlinks is set to False and the last element of the path is a
1477
symbolic link, chown will modify the link itself and not the file being
1478
referenced by the link.
1479
"""
1480
sys.audit('shutil.chown', path, user, group)
1481
1482
if user is None and group is None:
1483
raise ValueError("user and/or group must be set")
1484
1485
_user = user
1486
_group = group
1487
1488
# -1 means don't change it
1489
if user is None:
1490
_user = -1
1491
# user can either be an int (the uid) or a string (the system username)
1492
elif isinstance(user, str):
1493
_user = _get_uid(user)
1494
if _user is None:
1495
raise LookupError("no such user: {!r}".format(user))
1496
1497
if group is None:
1498
_group = -1
1499
elif not isinstance(group, int):
1500
_group = _get_gid(group)
1501
if _group is None:
1502
raise LookupError("no such group: {!r}".format(group))
1503
1504
os.chown(path, _user, _group, dir_fd=dir_fd,
1505
follow_symlinks=follow_symlinks)
1506
1507
def get_terminal_size(fallback=(80, 24)):
1508
"""Get the size of the terminal window.
1509
1510
For each of the two dimensions, the environment variable, COLUMNS
1511
and LINES respectively, is checked. If the variable is defined and
1512
the value is a positive integer, it is used.
1513
1514
When COLUMNS or LINES is not defined, which is the common case,
1515
the terminal connected to sys.__stdout__ is queried
1516
by invoking os.get_terminal_size.
1517
1518
If the terminal size cannot be successfully queried, either because
1519
the system doesn't support querying, or because we are not
1520
connected to a terminal, the value given in fallback parameter
1521
is used. Fallback defaults to (80, 24) which is the default
1522
size used by many terminal emulators.
1523
1524
The value returned is a named tuple of type os.terminal_size.
1525
"""
1526
# columns, lines are the working values
1527
try:
1528
columns = int(os.environ['COLUMNS'])
1529
except (KeyError, ValueError):
1530
columns = 0
1531
1532
try:
1533
lines = int(os.environ['LINES'])
1534
except (KeyError, ValueError):
1535
lines = 0
1536
1537
# only query if necessary
1538
if columns <= 0 or lines <= 0:
1539
try:
1540
size = os.get_terminal_size(sys.__stdout__.fileno())
1541
except (AttributeError, ValueError, OSError):
1542
# stdout is None, closed, detached, or not a terminal, or
1543
# os.get_terminal_size() is unsupported
1544
size = os.terminal_size(fallback)
1545
if columns <= 0:
1546
columns = size.columns or fallback[0]
1547
if lines <= 0:
1548
lines = size.lines or fallback[1]
1549
1550
return os.terminal_size((columns, lines))
1551
1552
1553
# Check that a given file can be accessed with the correct mode.
1554
# Additionally check that `file` is not a directory, as on Windows
1555
# directories pass the os.access check.
1556
def _access_check(fn, mode):
1557
return (os.path.exists(fn) and os.access(fn, mode)
1558
and not os.path.isdir(fn))
1559
1560
1561
def _win_path_needs_curdir(cmd, mode):
1562
"""
1563
On Windows, we can use NeedCurrentDirectoryForExePath to figure out
1564
if we should add the cwd to PATH when searching for executables if
1565
the mode is executable.
1566
"""
1567
return (not (mode & os.X_OK)) or _winapi.NeedCurrentDirectoryForExePath(
1568
os.fsdecode(cmd))
1569
1570
1571
def which(cmd, mode=os.F_OK | os.X_OK, path=None):
1572
"""Given a command, mode, and a PATH string, return the path which
1573
conforms to the given mode on the PATH, or None if there is no such
1574
file.
1575
1576
`mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
1577
of os.environ.get("PATH"), or can be overridden with a custom search
1578
path.
1579
1580
"""
1581
use_bytes = isinstance(cmd, bytes)
1582
1583
# If we're given a path with a directory part, look it up directly rather
1584
# than referring to PATH directories. This includes checking relative to
1585
# the current directory, e.g. ./script
1586
dirname, cmd = os.path.split(cmd)
1587
if dirname:
1588
path = [dirname]
1589
else:
1590
if path is None:
1591
path = os.environ.get("PATH", None)
1592
if path is None:
1593
try:
1594
path = os.confstr("CS_PATH")
1595
except (AttributeError, ValueError):
1596
# os.confstr() or CS_PATH is not available
1597
path = os.defpath
1598
# bpo-35755: Don't use os.defpath if the PATH environment variable
1599
# is set to an empty string
1600
1601
# PATH='' doesn't match, whereas PATH=':' looks in the current
1602
# directory
1603
if not path:
1604
return None
1605
1606
if use_bytes:
1607
path = os.fsencode(path)
1608
path = path.split(os.fsencode(os.pathsep))
1609
else:
1610
path = os.fsdecode(path)
1611
path = path.split(os.pathsep)
1612
1613
if sys.platform == "win32" and _win_path_needs_curdir(cmd, mode):
1614
curdir = os.curdir
1615
if use_bytes:
1616
curdir = os.fsencode(curdir)
1617
path.insert(0, curdir)
1618
1619
if sys.platform == "win32":
1620
# PATHEXT is necessary to check on Windows.
1621
pathext_source = os.getenv("PATHEXT") or _WIN_DEFAULT_PATHEXT
1622
pathext = pathext_source.split(os.pathsep)
1623
pathext = [ext.rstrip('.') for ext in pathext if ext]
1624
1625
if use_bytes:
1626
pathext = [os.fsencode(ext) for ext in pathext]
1627
1628
files = [cmd + ext for ext in pathext]
1629
1630
# If X_OK in mode, simulate the cmd.exe behavior: look at direct
1631
# match if and only if the extension is in PATHEXT.
1632
# If X_OK not in mode, simulate the first result of where.exe:
1633
# always look at direct match before a PATHEXT match.
1634
normcmd = cmd.upper()
1635
if not (mode & os.X_OK) or any(normcmd.endswith(ext.upper()) for ext in pathext):
1636
files.insert(0, cmd)
1637
else:
1638
# On other platforms you don't have things like PATHEXT to tell you
1639
# what file suffixes are executable, so just pass on cmd as-is.
1640
files = [cmd]
1641
1642
seen = set()
1643
for dir in path:
1644
normdir = os.path.normcase(dir)
1645
if normdir not in seen:
1646
seen.add(normdir)
1647
for thefile in files:
1648
name = os.path.join(dir, thefile)
1649
if _access_check(name, mode):
1650
return name
1651
return None
1652
1653
def __getattr__(name):
1654
if name == "ExecError":
1655
import warnings
1656
warnings._deprecated(
1657
"shutil.ExecError",
1658
f"{warnings._DEPRECATED_MSG}; it "
1659
"isn't raised by any shutil function.",
1660
remove=(3, 16)
1661
)
1662
return RuntimeError
1663
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")