
    @                     2   S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
Jr  SSK
Jr  SS	KJr  SS
KJr  \R                   " S5      r\R                   " S5      r\R                   " S5      r\R                   " S5      r\R                   " S5      r\R                   " S5      r\R                   " S5      r\" SS/5      r " S S\5      r " S S\5      r " S S\5      rS rS rS r S r!S r"S r#S  r$S(S! jr%S" r&S# r'S$ r(S% r)S& r*S' r+g))z*File and Cloud URL representation classes.    )absolute_import)print_function)division)unicode_literalsN)CommandException)InvalidUrlError)system_util)	text_utilz(?P<provider>[^:]*)://$z.(?P<provider>[^:]*)://(?P<bucket>[^/]*)/{0,1}$z6(?P<provider>[^:]*)://(?P<bucket>[^/]*)/(?P<object>.*)z&(?P<object>.+)#(?P<generation>[0-9]+)$z"(?P<object>.+)#(?P<version_id>.+)$z([^:]*://)(?P<filepath>.*)z[*?\[\]]...c                   r    \ rS rSrSrS rS rS rS rS r	SS	 jr
S
 r\S 5       r\S 5       rS rS rSrg)
StorageUrl3   z4Abstract base class for file and Cloud Storage URLs.c                     [        S5      e)NzClone not overriddenNotImplementedErrorselfs    $platform/gsutil/gslib/storage_url.pyCloneStorageUrl.Clone6   s    
4
55    c                     [        S5      e)NzIsFileUrl not overriddenr   r   s    r   	IsFileUrlStorageUrl.IsFileUrl9   s    
8
99r   c                     [        S5      e)NzIsCloudUrl not overriddenr   r   s    r   
IsCloudUrlStorageUrl.IsCloudUrl<   s    
9
::r   c                      [        S5      e)NzIsStream not overriddenr    r   r   IsStreamStorageUrl.IsStream?   s    
7
88r   c                     [        S5      e)NzIsFifo not overriddenr   r   s    r   IsFifoStorageUrl.IsFifoB   s    
5
66r   Nc                     [        S5      e)a  Returns a prefix of this URL that can be used for iterating.

Args:
  wildcard_suffix: If supplied, this wildcard suffix will be appended to the
                   prefix with a trailing slash before being returned.

Returns:
  A prefix of this URL that can be used for iterating.

If this URL contains a trailing slash, it will be stripped to create the
prefix. This helps avoid infinite looping when prefixes are iterated, but
preserves other slashes so that objects with '/' in the name are handled
properly.

For example, when recursively listing a bucket with the following contents:
  gs://bucket// <-- object named slash
  gs://bucket//one-dir-deep
a top-level expansion with '/' as a delimiter will result in the following
URL strings:
  'gs://bucket//' : OBJECT
  'gs://bucket//' : PREFIX
If we right-strip all slashes from the prefix entry and add a wildcard
suffix, we will get 'gs://bucket/*' which will produce identical results
(and infinitely recurse).

Example return values:
  ('gs://bucket/subdir/', '*') becomes 'gs://bucket/subdir/*'
  ('gs://bucket/', '*') becomes 'gs://bucket/*'
  ('gs://bucket/', None) becomes 'gs://bucket'
  ('gs://bucket/subdir//', '*') becomes 'gs://bucket/subdir//*'
  ('gs://bucket/subdir///', '**') becomes 'gs://bucket/subdir///**'
  ('gs://bucket/subdir/', '*') where 'subdir/' is an object becomes
       'gs://bucket/subdir/*', but iterating on this will return 'subdir/'
       as a BucketListingObject, so we will not recurse on it as a subdir
       during listing.
zCreatePrefixUrl not overriddenr   r   wildcard_suffixs     r   CreatePrefixUrlStorageUrl.CreatePrefixUrlE   s    J >
??r   c                 l   U R                   (       d  gU R                  U R                   -   U R                  -   nUR                  SR                  U R                  S95      nSR	                  U5      nSU;   a=  [
        R                  R                  SR                  [        R                  5      5        gg)z-Warn if ** use may lead to undefined results.Nz{delim}**{delim})delim z**zw** behavior is undefined if directly preceeded or followed by with characters other than / in the cloud and {} locally.)
object_namer,   splitformatjoinsysstderrwriteossep)r   delimiter_bounded_url	split_url+removed_correct_double_wildcards_url_strings       r    _WarnIfUnsupportedDoubleWildcard+StorageUrl._WarnIfUnsupportedDoubleWildcardl   s      JJ)9)99DJJF%++!!

!35I24'')2D/:: 
jjGGMvffH	 ;r   c                     [        S5      e)Nzurl_string not overriddenr   r   s    r   
url_stringStorageUrl.url_string   s    
9
::r   c                     [        S5      e)Nz%versionless_url_string not overriddenr   r   s    r   versionless_url_string!StorageUrl.versionless_url_string   s    
E
FFr   c                 b    [        U[        5      =(       a    U R                  UR                  :H  $ N)
isinstancer   r=   )r   others     r   __eq__StorageUrl.__eq__   s#    eZ(PT__@P@P-PPr   c                 ,    [        U R                  5      $ rC   )hashr=   r   s    r   __hash__StorageUrl.__hash__         r   r    rC   )__name__
__module____qualname____firstlineno____doc__r   r   r   r!   r$   r)   r:   propertyr=   r@   rF   rJ   __static_attributes__r    r   r   r   r   3   s_    <6:;97%@N, ; ; G GQ!r   r   c                   v    \ rS rSrSrSS jrS rS rS rS r	S r
S	 rSS jr\S 5       r\S 5       rS rSrg
)_FileUrl   aj  File URL class providing parsing and convenience methods.

This class assists with usage and manipulation of an
(optionally wildcarded) file URL string.  Depending on the string
contents, this class represents one or more directories or files.

For File URLs, scheme is always file, bucket_name is always blank,
and object_name contains the file/directory path.
c                    SU l         [        R                  U l        SU l        [
        R                  U5      nU(       a'  UR                  S:X  a  UR                  S5      U l	        OXl	        [        R                  (       a!  U R                  R                  SS5      U l	        S U l        X l        X0l        U R!                  5         g )Nfiler-      /\)schemer5   r6   r,   bucket_nameFILE_OBJECT_REGEXmatch	lastindexgroupr.   r	   
IS_WINDOWSreplace
generation	is_streamis_fifor:   )r   r=   re   rf   r_   s        r   __init___FileUrl.__init__   s    DKDJD ##J/EA%Qd# ))11#t<dDONL))+r   c                 ,    [        U R                  5      $ rC   )rU   r=   r   s    r   r   _FileUrl.Clone   s    DOO$$r   c                     gNTr    r   s    r   r   _FileUrl.IsFileUrl       r   c                     gNFr    r   s    r   r   _FileUrl.IsCloudUrl       r   c                     U R                   $ rC   )re   r   s    r   r!   _FileUrl.IsStream   s    >>r   c                     U R                   $ rC   )rf   r   s    r   r$   _FileUrl.IsFifo   s    <<r   c                     U R                  5       (       + =(       aE    U R                  5       (       + =(       a)    [        R                  R	                  U R
                  5      $ rC   )r!   r$   r5   pathisdirr.   r   s    r   IsDirectory_FileUrl.IsDirectory   s>     ,$5 ,GGMM$**+-r   Nc                     U R                   $ rC   r=   r'   s     r   r)   _FileUrl.CreatePrefixUrl       ??r   c                 <    U R                   < SU R                  < 3$ )N://)r\   r.   r   s    r   r=   _FileUrl.url_string   s    T%5%566r   c                     U R                   $ rC   r}   r   s    r   r@   _FileUrl.versionless_url_string   s    ??r   c                     U R                   $ rC   r}   r   s    r   __str___FileUrl.__str__   r   r   )r]   r,   rd   rf   re   r.   r\   )FFrC   )rM   rN   rO   rP   rQ   rg   r   r   r   r!   r$   rz   r)   rR   r=   r@   r   rS   r    r   r   rU   rU      s\    ,,%- 7 7  r   rU   c                       \ rS rSrSrS rS rS rS rS r	S r
S	 rS
 rS rS rSS jr\S 5       r\S 5       r\S 5       rS rSrg)	_CloudUrl   aU  Cloud URL class providing parsing and convenience methods.

This class assists with usage and manipulation of an
(optionally wildcarded) cloud URL string.  Depending on the string
contents, this class represents a provider, bucket(s), or object(s).

This class operates only on strings.  No cloud storage API calls are
made from this class.
c                    S U l         SU l        S U l        S U l        S U l        [
        R                  U5      n[        R                  U5      nU(       a  UR                  S5      U l         GOU(       a.  UR                  S5      U l         UR                  S5      U l        GOk[        R                  U5      nU(       Ga@  UR                  S5      U l         UR                  S5      U l        UR                  S5      U l        U R                  S:X  d  U R                  S:X  a  [        SU R                  -  5      eU R                   S:X  aS  [        R                  U R                  5      nU(       a,  UR                  S5      U l        UR                  S	5      U l        OqU R                   S
:X  aR  [        R                  U R                  5      nU(       a,  UR                  S5      U l        UR                  S5      U l        O[        SU-  5      eU[        U R                   5      [        S5      -   S  R                  U R                  5      (       a  [        SR                  U5      5      eU R!                  5         g )NrZ   providerbucketobjectr   r   z'%s is an invalid root-level object namegsrd   s3
version_idz/CloudUrl: URL string %s did not match URL regexr   zPCloud URL scheme should be followed by colon and two slashes: "://". Found: "{}")r\   r,   r]   r.   rd   PROVIDER_REGEXr_   BUCKET_REGEXra   OBJECT_REGEXr   GS_GENERATION_REGEXS3_VERSION_REGEXlen
startswithr0   r:   )r   r=   provider_matchbucket_matchobject_matchgeneration_matchversion_matchs          r   rg   _CloudUrl.__init__   s   DKDJDDDO#))*5N%%j1L"((4dk	 &&z2dk%++H5d!''
3l	"((4'--h7'--h7s"d&6&6$&> I $ 0 0!1 2 2;;$066t7G7GH
/55h?D.44\BDO[[D *001A1AB-,228<D+11,?DO=
JL 	L 3t{{#c%j023>>tzzJJ
+- - 	))+r   c                 ,    [        U R                  5      $ rC   )r   r=   r   s    r   r   _CloudUrl.Clone
  s    T__%%r   c                     grp   r    r   s    r   r   _CloudUrl.IsFileUrl  rr   r   c                     grl   r    r   s    r   r   _CloudUrl.IsCloudUrl  rn   r   c                     [        S5      e)Nz"IsStream not supported on CloudUrlr   r   s    r   r!   _CloudUrl.IsStream  s    
B
CCr   c                     [        S5      e)Nz IsFifo not supported on CloudUrlr   r   s    r   r$   _CloudUrl.IsFifo  s    
@
AAr   c                 \    [        U R                  =(       a    U R                  (       + 5      $ rC   boolr]   r.   r   s    r   IsBucket_CloudUrl.IsBucket  s!      9)9)9%9::r   c                 R    [        U R                  =(       a    U R                  5      $ rC   r   r   s    r   IsObject_CloudUrl.IsObject  s      5T%5%566r   c                 ,    [        U R                  5      $ rC   )r   rd   r   s    r   HasGeneration_CloudUrl.HasGeneration  rL   r   c                 \    [        U R                  =(       a    U R                  (       + 5      $ rC   )r   r\   r]   r   s    r   
IsProvider_CloudUrl.IsProvider"  s    4D$4$4 455r   Nc                 P    [        U R                  5      nU(       a	  U< SU< 3nU$ )NrZ   )StripOneSlashr@   )r   r(   prefixs      r   r)   _CloudUrl.CreatePrefixUrl%  s$    4667F /2fMr   c                 >    U R                   < SU R                  < S3$ )Nr   rZ   )r\   r]   r   s    r   bucket_url_string_CloudUrl.bucket_url_string+  s    d&6&677r   c                 l    U R                   nU R                  5       (       a  USU R                  -  -  nU$ )Nz#%s)r@   r   rd   )r   url_strs     r   r=   _CloudUrl.url_string/  s4    ))G((gNr   c                     U R                  5       (       a  SU R                  -  $ U R                  5       (       a  U R                  $ U R                  < SU R                  < SU R
                  < 3$ )Nz%s://r   rZ   )r   r\   r   r   r]   r.   r   s    r   r@    _CloudUrl.versionless_url_string6  sR    t{{""	###;;(8(8$:J:JKKr   c                     U R                   $ rC   r}   r   s    r   r   _CloudUrl.__str__>  r   r   )r]   r,   rd   r.   r\   rC   )rM   rN   rO   rP   rQ   rg   r   r   r   r!   r$   r   r   r   r   r)   rR   r   r=   r@   r   rS   r    r   r   r   r      s    ),V&DB;7!6 8 8   L Lr   r   c                 X    U R                  S5      nUS:X  a  gU SU R                  5       $ )z)Returns scheme component of a URL string.r   rX   r   )findlowerr   end_scheme_idxs     r   GetSchemeFromUrlStringr   B  s3     <<&.r1^$**,,r   c                     U S;   $ )N)rX   r   r   r    )
scheme_strs    r   IsKnownUrlSchemer   M  s    	+	++r   c                 B    U R                  S5      nUS:X  a  U $ XS-   S $ )z'Returns path component of a URL string.r   r      N)r   r   s     r   _GetPathFromUrlStringr   Q  s0     <<&.rNA%&''r   c                 >    [        [        R                  U 5      5      $ )zuChecks whether url_string contains a wildcard.

Args:
  url_string: URL string to check.

Returns:
  bool indicator.
)r   WILDCARD_REGEXsearchr}   s    r   ContainsWildcardr   [  s     
n##J/	00r   c                 `    U R                   S:X  a  U(       a  [        R                  " U5      $ U$ )a4  Decodes a generation from a StorageURL and a generation string.

This is used to represent gs and s3 versioning.

Args:
  url: StorageUrl representing the object.
  generation: Long or string representing the object's generation or
              version.

Returns:
  Valid generation string for use in URLs.
r   )r\   r
   DecodeLongAsString)urlrd   s     r   GenerationFromUrlAndStringr   g  s)     	ZZ4J''
33	r   c                 Z    U  H%  n[        U5      nUR                  5       (       d  M%    g   g)zChecks whether args_to_check contain any file URLs.

Args:
  args_to_check: Command-line argument subset to check.

Returns:
  True if args_to_check contains any file URLs.
TF)StorageUrlFromStringr   args_to_checkr   storage_urls      r   HaveFileUrlsr   y  s/     g&w/K  
r   c                     U  H<  n[        U5      nUR                  5       (       d  M%  UR                  5       (       d  M<    g   g)zChecks whether args_to_check contains any provider URLs (like 'gs://').

Args:
  args_to_check: Command-line argument subset to check.

Returns:
  True if args_to_check contains any provider URLs.
TF)r   r   r   r   s      r   HaveProviderUrlsr     s=     g&w/KK$:$:$<$<  
r   c                    U R                  5       (       d  gU R                  nUR                  S5      (       a  gU(       a,  UR                  5       (       a  UR                  R
                  nOSnUS:H  =(       a    UR                  S5      $ )a:  Determines if a StorageUrl is a cloud subdir placeholder.

This function is needed because GUI tools (like the GCS cloud console) allow
users to create empty "folders" by creating a placeholder object; and parts
of gsutil need to treat those placeholder objects specially. For example,
gsutil rsync needs to avoid downloading those objects because they can cause
conflicts (see comments in rsync command for details).

We currently detect two cases:
  - Cloud objects whose name ends with '_$folder$'
  - Cloud objects whose name ends with '/'

Args:
  url: (gslib.storage_url.StorageUrl) The URL to be checked.
  blr: (gslib.BucketListingRef or None) The blr to check, or None if not
      available. If `blr` is None, size won't be checked.

Returns:
  (bool) True if the URL is a cloud subdir placeholder, otherwise False.
Fz	_$folder$Tr   rZ   )r   r=   endswithr   root_objectsize)r   blrr   r   s       r   IsCloudSubdirPlaceholderr     sl    * 
		NN'k""S\\^^??DD		,w'',,r   c                     [        U 5      S:H  $ )z'Returns whether a string is a file URL.rX   )r   r   s    r   IsFileUrlStringr     s     
 	(F	22r   c                 8   [        U 5      n[        U5      (       d  [        SU-  5      eUS:X  aQ  [        U 5      nUS:H  nSn [        R
                  " [        R                  " U5      R                  5      n[        XUS9$ [        U 5      $ ! [         a     N!f = f)z@Static factory function for creating a StorageUrl from a string.zUnrecognized scheme "%s"rX   -F)re   rf   )r   r   r   r   statS_ISFIFOr5   st_modeOSErrorrU   r   )r   r\   rx   re   rf   s        r   r   r     s     "'*&	&	!	!
4v=
>>v )DIGbggdm334g G'BB	7	  
s   4B 
BBc                 J    U (       a  U R                  S5      (       a  U S S $ U $ )NrZ   r   )r   r   s    r   r   r     s&    !!#&&3B<	.r   c                     SnSnU  H5  n[        U5      nU(       d  UR                  nM#  UR                  U:w  d  M5    g   USL$ )a4  Tests whether the URLs are all for a single provider.

Args:
  url_args: (Iterable[str]) Collection of strings to check.

Returns:
  True if all URLs are for single provider; False if `url_args` was empty (as
  this would not result in a single unique provider) or URLs targeted multiple
  unique providers.
NF)r   r\   )url_argsr   r   r   s       r   UrlsAreForSingleProviderr     sJ     (#g
w
'Ch	x	  
	r   c                     [        S U  5       5      (       a8  [        [        S U 5      5      n[        U5      =(       a    [        U5      (       + $ g)a(  Tests whether the URLs are a mix of buckets and objects.

Args:
  url_args: (Iterable[gslib.storage_url.StorageUrl]) Collection of URLs to
  check.

Returns:
  True if URLs are a mix of buckets and objects. False if URLs are all buckets
  or all objects. None if invalid Cloud URLs are included.
c              3   @   #    U  H  oR                  5       v   M     g 7frC   )r   ).0r   s     r   	<genexpr>0UrlsAreMixOfBucketsAndObjects.<locals>.<genexpr>  s     *Tc		Ts   c                 "    U R                  5       $ rC   )r   )xs    r   <lambda>/UrlsAreMixOfBucketsAndObjects.<locals>.<lambda>  s
    QZZ\r   N)alllistmapany)urlsare_bucketss     r   UrlsAreMixOfBucketsAndObjectsr    sD     	*T***s1489K{4C$4 44 +r   c                 J    [        U 5      (       a  U(       d  [        S5      egg)zBRaises error if mix of buckets and objects adjusted for recursion.z/Cannot operate on a mix of buckets and objects.N)r  r   )r  recursion_requesteds     r   )RaiseErrorIfUrlsAreMixOfBucketsAndObjectsr    s%    "4((1D
L
MM 2E(r   rC   ),rQ   
__future__r   r   r   r   r5   rer   r2   gslib.exceptionr   r   gslib.utilsr	   r
   compiler   r   r   r   r   r^   r   	frozensetRELATIVE_PATH_SYMBOLSr   r   rU   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r    r   r   <module>r     s#   1 & %  ' 	 	  
 , + # ! 67zzKLzz=? jj!JK ::CD JJ<= K(!3+. [! [!|@z @Fk
 k\-,(	1$  -B3&,5 Nr   