-
Notifications
You must be signed in to change notification settings - Fork 48
/
cloudpath.py
1337 lines (1088 loc) · 46.9 KB
/
cloudpath.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import abc
from collections import defaultdict
import collections.abc
from contextlib import contextmanager
import os
from pathlib import ( # type: ignore
Path,
PosixPath,
PurePosixPath,
WindowsPath,
_PathParents,
)
import shutil
import sys
from typing import (
overload,
Any,
Callable,
Container,
Iterable,
IO,
Dict,
Generator,
List,
Optional,
Sequence,
Tuple,
Type,
TYPE_CHECKING,
TypeVar,
Union,
)
from urllib.parse import urlparse
from warnings import warn
if sys.version_info >= (3, 10):
from typing import TypeGuard
else:
from typing_extensions import TypeGuard
if sys.version_info >= (3, 11):
from typing import Self
else:
from typing_extensions import Self
if sys.version_info >= (3, 12):
from pathlib import posixpath as _posix_flavour # type: ignore[attr-defined]
from pathlib import _make_selector # type: ignore[attr-defined]
else:
from pathlib import _posix_flavour # type: ignore[attr-defined]
from pathlib import _make_selector as _make_selector_pathlib # type: ignore[attr-defined]
def _make_selector(pattern_parts, _flavour, case_sensitive=True):
return _make_selector_pathlib(tuple(pattern_parts), _flavour)
from cloudpathlib.enums import FileCacheMode
from . import anypath
from .exceptions import (
ClientMismatchError,
CloudPathFileExistsError,
CloudPathIsADirectoryError,
CloudPathNotADirectoryError,
CloudPathNotImplementedError,
DirectoryNotEmptyError,
IncompleteImplementationError,
InvalidPrefixError,
MissingDependenciesError,
NoStatError,
OverwriteDirtyFileError,
OverwriteNewerCloudError,
OverwriteNewerLocalError,
)
if TYPE_CHECKING:
from .client import Client
class CloudImplementation:
name: str
dependencies_loaded: bool = True
_client_class: Type["Client"]
_path_class: Type["CloudPath"]
def validate_completeness(self) -> None:
expected = ["client_class", "path_class"]
missing = [cls for cls in expected if getattr(self, f"_{cls}") is None]
if missing:
raise IncompleteImplementationError(
f"Implementation is missing registered components: {missing}"
)
if not self.dependencies_loaded:
raise MissingDependenciesError(
f"Missing dependencies for {self._client_class.__name__}. You can install them "
f"with 'pip install cloudpathlib[{self.name}]'."
)
@property
def client_class(self) -> Type["Client"]:
self.validate_completeness()
return self._client_class
@property
def path_class(self) -> Type["CloudPath"]:
self.validate_completeness()
return self._path_class
implementation_registry: Dict[str, CloudImplementation] = defaultdict(CloudImplementation)
T = TypeVar("T")
CloudPathT = TypeVar("CloudPathT", bound="CloudPath")
def register_path_class(key: str) -> Callable[[Type[CloudPathT]], Type[CloudPathT]]:
def decorator(cls: Type[CloudPathT]) -> Type[CloudPathT]:
if not issubclass(cls, CloudPath):
raise TypeError("Only subclasses of CloudPath can be registered.")
implementation_registry[key]._path_class = cls
cls._cloud_meta = implementation_registry[key]
return cls
return decorator
class CloudPathMeta(abc.ABCMeta):
@overload
def __call__(cls: Type[T], cloud_path: CloudPathT, *args: Any, **kwargs: Any) -> CloudPathT:
...
@overload
def __call__(
cls: Type[T], cloud_path: Union[str, "CloudPath"], *args: Any, **kwargs: Any
) -> T:
...
def __call__(
cls: Type[T], cloud_path: Union[str, CloudPathT], *args: Any, **kwargs: Any
) -> Union[T, "CloudPath", CloudPathT]:
# cls is a class that is the instance of this metaclass, e.g., CloudPath
if not issubclass(cls, CloudPath):
raise TypeError(
f"Only subclasses of {CloudPath.__name__} can be instantiated from its meta class."
)
# Dispatch to subclass if base CloudPath
if cls is CloudPath:
for implementation in implementation_registry.values():
path_class = implementation._path_class
if path_class is not None and path_class.is_valid_cloudpath(
cloud_path, raise_on_error=False
):
# Instantiate path_class instance
new_obj = object.__new__(path_class)
path_class.__init__(new_obj, cloud_path, *args, **kwargs) # type: ignore[type-var]
return new_obj
valid_prefixes = [
impl._path_class.cloud_prefix
for impl in implementation_registry.values()
if impl._path_class is not None
]
raise InvalidPrefixError(
f"Path {cloud_path} does not begin with a known prefix {valid_prefixes}."
)
new_obj = object.__new__(cls)
cls.__init__(new_obj, cloud_path, *args, **kwargs) # type: ignore[type-var]
return new_obj
def __init__(cls, name: str, bases: Tuple[type, ...], dic: Dict[str, Any]) -> None:
# Copy docstring from pathlib.Path
for attr in dir(cls):
if (
not attr.startswith("_")
and hasattr(Path, attr)
and getattr(getattr(Path, attr), "__doc__", None)
):
docstring = getattr(Path, attr).__doc__ + " _(Docstring copied from pathlib.Path)_"
getattr(cls, attr).__doc__ = docstring
if isinstance(getattr(cls, attr), property):
# Properties have __doc__ duplicated under fget, and at least some parsers
# read it from there.
getattr(cls, attr).fget.__doc__ = docstring
# Abstract base class
class CloudPath(metaclass=CloudPathMeta):
"""Base class for cloud storage file URIs, in the style of the Python standard library's
[`pathlib` module](https://docs.python.org/3/library/pathlib.html). Instances represent a path
in cloud storage with filesystem path semantics, and convenient methods allow for basic
operations like joining, reading, writing, iterating over contents, etc. `CloudPath` almost
entirely mimics the [`pathlib.Path`](https://docs.python.org/3/library/pathlib.html#pathlib.Path)
interface, so most familiar properties and methods should be available and behave in the
expected way.
Analogous to the way `pathlib.Path` works, instantiating `CloudPath` will instead create an
instance of an appropriate subclass that implements a particular cloud storage service, such as
[`S3Path`](../s3path). This dispatching behavior is based on the URI scheme part of a cloud
storage URI (e.g., `"s3://"`).
"""
_cloud_meta: CloudImplementation
cloud_prefix: str
def __init__(
self,
cloud_path: Union[str, Self, "CloudPath"],
client: Optional["Client"] = None,
) -> None:
# handle if local file gets opened. must be set at the top of the method in case any code
# below raises an exception, this prevents __del__ from raising an AttributeError
self._handle: Optional[IO] = None
self.is_valid_cloudpath(cloud_path, raise_on_error=True)
# versions of the raw string that provide useful methods
self._str = str(cloud_path)
self._url = urlparse(self._str)
self._path = PurePosixPath(f"/{self._no_prefix}")
# setup client
if client is None:
if isinstance(cloud_path, CloudPath):
client = cloud_path.client
else:
client = self._cloud_meta.client_class.get_default_client()
if not isinstance(client, self._cloud_meta.client_class):
raise ClientMismatchError(
f"Client of type [{client.__class__}] is not valid for cloud path of type "
f"[{self.__class__}]; must be instance of [{self._cloud_meta.client_class}], or "
f"None to use default client for this cloud path class."
)
self.client: Client = client
# track if local has been written to, if so it may need to be uploaded
self._dirty = False
def __del__(self) -> None:
# make sure that file handle to local path is closed
if self._handle is not None:
self._handle.close()
# ensure file removed from cache when cloudpath object deleted
if (
hasattr(self, "client")
and self.client.file_cache_mode == FileCacheMode.cloudpath_object
):
self.clear_cache()
def __getstate__(self) -> Dict[str, Any]:
state = self.__dict__.copy()
# don't pickle client
del state["client"]
return state
def __setstate__(self, state: Dict[str, Any]) -> None:
client = self._cloud_meta.client_class.get_default_client()
state["client"] = client
self.__dict__.update(state)
@property
def _no_prefix(self) -> str:
return self._str[len(self.cloud_prefix) :]
@property
def _no_prefix_no_drive(self) -> str:
return self._str[len(self.cloud_prefix) + len(self.drive) :]
@overload
@classmethod
def is_valid_cloudpath(cls, path: "CloudPath", raise_on_error: bool = ...) -> TypeGuard[Self]:
...
@overload
@classmethod
def is_valid_cloudpath(cls, path: str, raise_on_error: bool = ...) -> bool:
...
@classmethod
def is_valid_cloudpath(
cls, path: Union[str, "CloudPath"], raise_on_error: bool = False
) -> Union[bool, TypeGuard[Self]]:
valid = str(path).lower().startswith(cls.cloud_prefix.lower())
if raise_on_error and not valid:
raise InvalidPrefixError(
f"'{path}' is not a valid path since it does not start with '{cls.cloud_prefix}'"
)
return valid
def __repr__(self) -> str:
return f"{self.__class__.__name__}('{self}')"
def __str__(self) -> str:
return self._str
def __hash__(self) -> int:
return hash((type(self).__name__, str(self)))
def __eq__(self, other: Any) -> bool:
return isinstance(other, type(self)) and str(self) == str(other)
def __fspath__(self) -> str:
if self.is_file():
self._refresh_cache(force_overwrite_from_cloud=False)
return str(self._local)
def __lt__(self, other: Any) -> bool:
if not isinstance(other, type(self)):
return NotImplemented
return self.parts < other.parts
def __le__(self, other: Any) -> bool:
if not isinstance(other, type(self)):
return NotImplemented
return self.parts <= other.parts
def __gt__(self, other: Any) -> bool:
if not isinstance(other, type(self)):
return NotImplemented
return self.parts > other.parts
def __ge__(self, other: Any) -> bool:
if not isinstance(other, type(self)):
return NotImplemented
return self.parts >= other.parts
# ====================== NOT IMPLEMENTED ======================
# as_posix - no cloud equivalent; not needed since we assume url separator
# chmod - permission changing should be explicitly done per client with methods
# that make sense for the client permission options
# cwd - no cloud equivalent
# expanduser - no cloud equivalent
# group - should be implemented with client-specific permissions
# home - no cloud equivalent
# is_block_device - no cloud equivalent
# is_char_device - no cloud equivalent
# is_fifo - no cloud equivalent
# is_mount - no cloud equivalent
# is_reserved - no cloud equivalent
# is_socket - no cloud equivalent
# is_symlink - no cloud equivalent
# lchmod - no cloud equivalent
# lstat - no cloud equivalent
# owner - no cloud equivalent
# root - drive already has the bucket and anchor/prefix has the scheme, so nothing to store here
# symlink_to - no cloud equivalent
# link_to - no cloud equivalent
# hardlink_to - no cloud equivalent
# ====================== REQUIRED, NOT GENERIC ======================
# Methods that must be implemented, but have no generic application
@property
@abc.abstractmethod
def drive(self) -> str:
"""For example "bucket" on S3 or "container" on Azure; needs to be defined for each class"""
pass
@abc.abstractmethod
def is_dir(self) -> bool:
"""Should be implemented without requiring a dir is downloaded"""
pass
@abc.abstractmethod
def is_file(self) -> bool:
"""Should be implemented without requiring that the file is downloaded"""
pass
@abc.abstractmethod
def mkdir(self, parents: bool = False, exist_ok: bool = False) -> None:
"""Should be implemented using the client API without requiring a dir is downloaded"""
pass
@abc.abstractmethod
def touch(self, exist_ok: bool = True) -> None:
"""Should be implemented using the client API to create and update modified time"""
pass
# ====================== IMPLEMENTED FROM SCRATCH ======================
# Methods with their own implementations that work generically
def __rtruediv__(self, other: Any) -> None:
raise ValueError(
"Cannot change a cloud path's root since all paths are absolute; create a new path instead."
)
@property
def anchor(self) -> str:
return self.cloud_prefix
def as_uri(self) -> str:
return str(self)
def exists(self) -> bool:
return self.client._exists(self)
@property
def fspath(self) -> str:
return self.__fspath__()
def _glob_checks(self, pattern: str) -> None:
if ".." in pattern:
raise CloudPathNotImplementedError(
"Relative paths with '..' not supported in glob patterns."
)
if pattern.startswith(self.cloud_prefix) or pattern.startswith("/"):
raise CloudPathNotImplementedError("Non-relative patterns are unsupported")
if self.drive == "":
raise CloudPathNotImplementedError(
".glob is only supported within a bucket or container; you can use `.iterdir` to list buckets; for example, CloudPath('s3://').iterdir()"
)
def _build_subtree(self, recursive):
# build a tree structure for all files out of default dicts
Tree: Callable = lambda: defaultdict(Tree)
def _build_tree(trunk, branch, nodes, is_dir):
"""Utility to build a tree from nested defaultdicts with a generator
of nodes (parts) of a path."""
next_branch = next(nodes, None)
if next_branch is None:
trunk[branch] = Tree() if is_dir else None # leaf node
else:
_build_tree(trunk[branch], next_branch, nodes, is_dir)
file_tree = Tree()
for f, is_dir in self.client._list_dir(self, recursive=recursive):
parts = str(f.relative_to(self)).split("/")
# skip self
if len(parts) == 1 and parts[0] == ".":
continue
nodes = (p for p in parts)
_build_tree(file_tree, next(nodes, None), nodes, is_dir)
return dict(file_tree) # freeze as normal dict before passing in
def _glob(self, selector, recursive: bool) -> Generator[Self, None, None]:
file_tree = self._build_subtree(recursive)
root = _CloudPathSelectable(
self.name,
[], # nothing above self will be returned, so initial parents is empty
file_tree,
)
for p in selector.select_from(root):
# select_from returns self.name/... so strip before joining
yield (self / str(p)[len(self.name) + 1 :])
def glob(
self, pattern: str, case_sensitive: Optional[bool] = None
) -> Generator[Self, None, None]:
self._glob_checks(pattern)
pattern_parts = PurePosixPath(pattern).parts
selector = _make_selector(
tuple(pattern_parts), _posix_flavour, case_sensitive=case_sensitive
)
yield from self._glob(
selector,
"/" in pattern
or "**"
in pattern, # recursive listing needed if explicit ** or any sub folder in pattern
)
def rglob(
self, pattern: str, case_sensitive: Optional[bool] = None
) -> Generator[Self, None, None]:
self._glob_checks(pattern)
pattern_parts = PurePosixPath(pattern).parts
selector = _make_selector(
("**",) + tuple(pattern_parts), _posix_flavour, case_sensitive=case_sensitive
)
yield from self._glob(selector, True)
def iterdir(self) -> Generator[Self, None, None]:
for f, _ in self.client._list_dir(self, recursive=False):
if f != self: # iterdir does not include itself in pathlib
yield f
@staticmethod
def _walk_results_from_tree(root, tree, top_down=True):
"""Utility to yield tuples in the form expected by `.walk` from the file
tree constructed by `_build_substree`.
"""
dirs = []
files = []
for item, branch in tree.items():
files.append(item) if branch is None else dirs.append(item)
if top_down:
yield root, dirs, files
for dir in dirs:
yield from CloudPath._walk_results_from_tree(root / dir, tree[dir], top_down=top_down)
if not top_down:
yield root, dirs, files
def walk(
self,
top_down: bool = True,
on_error: Optional[Callable] = None,
follow_symlinks: bool = False,
) -> Generator[Tuple[Self, List[str], List[str]], None, None]:
try:
file_tree = self._build_subtree(recursive=True) # walking is always recursive
yield from self._walk_results_from_tree(self, file_tree, top_down=top_down)
except Exception as e:
if on_error is not None:
on_error(e)
else:
raise
def open(
self,
mode: str = "r",
buffering: int = -1,
encoding: Optional[str] = None,
errors: Optional[str] = None,
newline: Optional[str] = None,
force_overwrite_from_cloud: bool = False, # extra kwarg not in pathlib
force_overwrite_to_cloud: bool = False, # extra kwarg not in pathlib
) -> IO[Any]:
# if trying to call open on a directory that exists
if self.exists() and not self.is_file():
raise CloudPathIsADirectoryError(
f"Cannot open directory, only files. Tried to open ({self})"
)
if mode == "x" and self.exists():
raise CloudPathFileExistsError(f"Cannot open existing file ({self}) for creation.")
# TODO: consider streaming from client rather than DLing entire file to cache
self._refresh_cache(force_overwrite_from_cloud=force_overwrite_from_cloud)
# create any directories that may be needed if the file is new
if not self._local.exists():
self._local.parent.mkdir(parents=True, exist_ok=True)
original_mtime = 0.0
else:
original_mtime = self._local.stat().st_mtime
buffer = self._local.open(
mode=mode,
buffering=buffering,
encoding=encoding,
errors=errors,
newline=newline,
)
# write modes need special on closing the buffer
if any(m in mode for m in ("w", "+", "x", "a")):
# dirty, handle, patch close
wrapped_close = buffer.close
# since we are pretending this is a cloud file, upload it to the cloud
# when the buffer is closed
def _patched_close_upload(*args, **kwargs) -> None:
wrapped_close(*args, **kwargs)
# we should be idempotent and not upload again if
# we already ran our close method patch
if not self._dirty:
return
# original mtime should match what was in the cloud; because of system clocks or rounding
# by the cloud provider, the new version in our cache is "older" than the original version;
# explicitly set the new modified time to be after the original modified time.
if self._local.stat().st_mtime < original_mtime:
new_mtime = original_mtime + 1
os.utime(self._local, times=(new_mtime, new_mtime))
self._upload_local_to_cloud(force_overwrite_to_cloud=force_overwrite_to_cloud)
self._dirty = False
buffer.close = _patched_close_upload # type: ignore
# keep reference in case we need to close when __del__ is called on this object
self._handle = buffer
# opened for write, so mark dirty
self._dirty = True
# if we don't want any cache around, remove the cache
# as soon as the file is closed
if self.client.file_cache_mode == FileCacheMode.close_file:
# this may be _patched_close_upload, in which case we need to
# make sure to call that first so the file gets uploaded
wrapped_close_for_cache = buffer.close
def _patched_close_empty_cache(*args, **kwargs):
wrapped_close_for_cache(*args, **kwargs)
# remove local file as last step on closing
self.clear_cache()
buffer.close = _patched_close_empty_cache # type: ignore
return buffer
def replace(self, target: Self) -> Self:
if type(self) is not type(target):
raise TypeError(
f"The target based to rename must be an instantiated class of type: {type(self)}"
)
if self.is_dir():
raise CloudPathIsADirectoryError(
f"Path {self} is a directory; rename/replace the files recursively."
)
if target == self:
# Request is to replace/rename this with the same path - nothing to do
return self
if target.exists():
target.unlink()
self.client._move_file(self, target)
return target
def rename(self, target: Self) -> Self:
# for cloud services replace == rename since we don't just rename,
# we actually move files
return self.replace(target)
def rmdir(self) -> None:
if self.is_file():
raise CloudPathNotADirectoryError(
f"Path {self} is a file; call unlink instead of rmdir."
)
try:
next(self.iterdir())
raise DirectoryNotEmptyError(
f"Directory not empty: '{self}'. Use rmtree to delete recursively."
)
except StopIteration:
pass
self.client._remove(self)
def samefile(self, other_path: Union[str, os.PathLike]) -> bool:
# all cloud paths are absolute and the paths are used for hash
return self == other_path
def unlink(self, missing_ok: bool = True) -> None:
# Note: missing_ok defaults to False in pathlib, but changing the default now would be a breaking change.
if self.is_dir():
raise CloudPathIsADirectoryError(
f"Path {self} is a directory; call rmdir instead of unlink."
)
self.client._remove(self, missing_ok)
def write_bytes(self, data: bytes) -> int:
"""Open the file in bytes mode, write to it, and close the file.
NOTE: vendored from pathlib since we override open
https://github.com/python/cpython/blob/3.8/Lib/pathlib.py#L1235-L1242
"""
# type-check for the buffer interface before truncating the file
view = memoryview(data)
with self.open(mode="wb") as f:
return f.write(view)
def write_text(
self,
data: str,
encoding: Optional[str] = None,
errors: Optional[str] = None,
newline: Optional[str] = None,
) -> int:
"""Open the file in text mode, write to it, and close the file.
NOTE: vendored from pathlib since we override open
https://github.com/python/cpython/blob/3.10/Lib/pathlib.py#L1146-L1155
"""
if not isinstance(data, str):
raise TypeError("data must be str, not %s" % data.__class__.__name__)
with self.open(mode="w", encoding=encoding, errors=errors, newline=newline) as f:
return f.write(data)
def read_bytes(self) -> bytes:
with self.open(mode="rb") as f:
return f.read()
def read_text(self, encoding: Optional[str] = None, errors: Optional[str] = None) -> str:
with self.open(mode="r", encoding=encoding, errors=errors) as f:
return f.read()
def is_junction(self):
return False # only windows paths can be junctions, not cloudpaths
# ====================== DISPATCHED TO POSIXPATH FOR PURE PATHS ======================
# Methods that are dispatched to exactly how pathlib.PurePosixPath would calculate it on
# self._path for pure paths (does not matter if file exists);
# see the next session for ones that require a real file to exist
def _dispatch_to_path(self, func: str, *args, **kwargs) -> Any:
"""Some functions we can just dispatch to the pathlib version
We want to do this explicitly so we don't have to support all
of pathlib and subclasses can override individually if necessary.
"""
path_version = self._path.__getattribute__(func)
# Path functions should be called so the results are calculated
if callable(path_version):
path_version = path_version(*args, **kwargs)
# Paths should always be resolved and then converted to the same client + class as this one
if isinstance(path_version, PurePosixPath):
# always resolve since cloud paths must be absolute
path_version = _resolve(path_version)
return self._new_cloudpath(path_version)
# When sequence of PurePosixPath, we want to convert to sequence of CloudPaths
if (
isinstance(path_version, collections.abc.Sequence)
and len(path_version) > 0
and isinstance(path_version[0], PurePosixPath)
):
sequence_class = (
type(path_version) if not isinstance(path_version, _PathParents) else tuple
)
return sequence_class( # type: ignore
self._new_cloudpath(_resolve(p)) for p in path_version if _resolve(p) != p.root
)
# when pathlib returns something else, we probably just want that thing
# cases this should include: str, empty sequence, sequence of str, ...
else:
return path_version
def __truediv__(self, other: Union[str, PurePosixPath]) -> Self:
if not isinstance(other, (str, PurePosixPath)):
raise TypeError(f"Can only join path {repr(self)} with strings or posix paths.")
return self._dispatch_to_path("__truediv__", other)
def joinpath(self, *pathsegments: Union[str, os.PathLike]) -> Self:
return self._dispatch_to_path("joinpath", *pathsegments)
def absolute(self) -> Self:
return self
def is_absolute(self) -> bool:
return True
def resolve(self, strict: bool = False) -> Self:
return self
def relative_to(self, other: Self, walk_up: bool = False) -> PurePosixPath:
# We don't dispatch regularly since this never returns a cloud path (since it is relative, and cloud paths are
# absolute)
if not isinstance(other, CloudPath):
raise ValueError(f"{self} is a cloud path, but {other} is not")
if self.cloud_prefix != other.cloud_prefix:
raise ValueError(
f"{self} is a {self.cloud_prefix} path, but {other} is a {other.cloud_prefix} path"
)
kwargs = dict(walk_up=walk_up)
if sys.version_info < (3, 12):
kwargs.pop("walk_up")
return self._path.relative_to(other._path, **kwargs) # type: ignore[call-arg]
def is_relative_to(self, other: Self) -> bool:
try:
self.relative_to(other)
return True
except ValueError:
return False
@property
def name(self) -> str:
return self._dispatch_to_path("name")
def match(self, path_pattern: str, case_sensitive: Optional[bool] = None) -> bool:
# strip scheme from start of pattern before testing
if path_pattern.startswith(self.anchor + self.drive + "/"):
path_pattern = path_pattern[len(self.anchor + self.drive + "/") :]
kwargs = dict(case_sensitive=case_sensitive)
if sys.version_info < (3, 12):
kwargs.pop("case_sensitive")
return self._dispatch_to_path("match", path_pattern, **kwargs)
@property
def parent(self) -> Self:
return self._dispatch_to_path("parent")
@property
def parents(self) -> Sequence[Self]:
return self._dispatch_to_path("parents")
@property
def parts(self) -> Tuple[str, ...]:
parts = self._dispatch_to_path("parts")
if parts[0] == "/":
parts = parts[1:]
return (self.anchor, *parts)
@property
def stem(self) -> str:
return self._dispatch_to_path("stem")
@property
def suffix(self) -> str:
return self._dispatch_to_path("suffix")
@property
def suffixes(self) -> List[str]:
return self._dispatch_to_path("suffixes")
def with_stem(self, stem: str) -> Self:
try:
return self._dispatch_to_path("with_stem", stem)
except AttributeError:
# with_stem was only added in python 3.9, so we fallback for compatibility
return self.with_name(stem + self.suffix)
def with_name(self, name: str) -> Self:
return self._dispatch_to_path("with_name", name)
def with_segments(self, *pathsegments) -> Self:
"""Create a new CloudPath with the same client out of the given segments.
The first segment will be interpreted as the bucket/container name.
"""
return self._new_cloudpath("/".join(pathsegments))
def with_suffix(self, suffix: str) -> Self:
return self._dispatch_to_path("with_suffix", suffix)
# ====================== DISPATCHED TO LOCAL CACHE FOR CONCRETE PATHS ======================
# Items that can be executed on the cached file on the local filesystem
def _dispatch_to_local_cache_path(self, func: str, *args, **kwargs) -> Any:
self._refresh_cache()
path_version = self._local.__getattribute__(func)
# Path functions should be called so the results are calculated
if callable(path_version):
path_version = path_version(*args, **kwargs)
# Paths should always be resolved and then converted to the same client + class as this one
if isinstance(path_version, (PosixPath, WindowsPath)):
# always resolve since cloud paths must be absolute
path_version = path_version.resolve()
return self._new_cloudpath(path_version)
# when pathlib returns a string, etc. we probably just want that thing
else:
return path_version
def stat(self, follow_symlinks: bool = True) -> os.stat_result:
"""Note: for many clients, we may want to override so we don't incur
network costs since many of these properties are available as
API calls.
"""
warn(
f"stat not implemented as API call for {self.__class__} so file must be downloaded to "
f"calculate stats; this may take a long time depending on filesize"
)
return self._dispatch_to_local_cache_path("stat", follow_symlinks=follow_symlinks)
# =========== public cloud methods, not in pathlib ===============
def download_to(self, destination: Union[str, os.PathLike]) -> Path:
destination = Path(destination)
if self.is_file():
if destination.is_dir():
destination = destination / self.name
return self.client._download_file(self, destination)
else:
destination.mkdir(exist_ok=True)
for f in self.iterdir():
rel = str(self)
if not rel.endswith("/"):
rel = rel + "/"
rel_dest = str(f)[len(rel) :]
f.download_to(destination / rel_dest)
return destination
def rmtree(self) -> None:
"""Delete an entire directory tree."""
if self.is_file():
raise CloudPathNotADirectoryError(
f"Path {self} is a file; call unlink instead of rmtree."
)
self.client._remove(self)
def upload_from(
self,
source: Union[str, os.PathLike],
force_overwrite_to_cloud: bool = False,
) -> Self:
"""Upload a file or directory to the cloud path."""
source = Path(source)
if source.is_dir():
for p in source.iterdir():
(self / p.name).upload_from(p, force_overwrite_to_cloud=force_overwrite_to_cloud)
return self
else:
if self.exists() and self.is_dir():
dst = self / source.name
else:
dst = self
dst._upload_file_to_cloud(source, force_overwrite_to_cloud=force_overwrite_to_cloud)
return dst
@overload
def copy(
self,
destination: Self,
force_overwrite_to_cloud: bool = False,
) -> Self:
...
@overload
def copy(
self,
destination: Path,
force_overwrite_to_cloud: bool = False,
) -> Path:
...
@overload
def copy(
self,
destination: str,
force_overwrite_to_cloud: bool = False,
) -> Union[Path, "CloudPath"]:
...
def copy(self, destination, force_overwrite_to_cloud=False):
"""Copy self to destination folder of file, if self is a file."""
if not self.exists() or not self.is_file():
raise ValueError(
f"Path {self} should be a file. To copy a directory tree use the method copytree."
)
# handle string version of cloud paths + local paths
if isinstance(destination, (str, os.PathLike)):
destination = anypath.to_anypath(destination)
if not isinstance(destination, CloudPath):
return self.download_to(destination)
# if same client, use cloud-native _move_file on client to avoid downloading
if self.client is destination.client:
if destination.exists() and destination.is_dir():
destination = destination / self.name
if (
not force_overwrite_to_cloud
and destination.exists()
and destination.stat().st_mtime >= self.stat().st_mtime
):
raise OverwriteNewerCloudError(
f"File ({destination}) is newer than ({self}). "
f"To overwrite "
f"pass `force_overwrite_to_cloud=True`."
)
return self.client._move_file(self, destination, remove_src=False)
else:
if not destination.exists() or destination.is_file():
return destination.upload_from(
self.fspath, force_overwrite_to_cloud=force_overwrite_to_cloud
)
else:
return (destination / self.name).upload_from(