
    V                    D   S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKJ	r	  SSK
r
SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJs  Jr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr   SSKJ!r!  SSKJ"r"  SSKJ#r#  SSKJ$r$  SSKJ%r%  SSKJ&r&  SSKJ'r'  SSKJ(r(  SSKJ)r)  SSKJr  SSK*J+r+  SSK,J-r-  SS K.J/r/  SS!K.J0r0  SS"K.J1r1  SS#K.J2r2  SS$K.J3r3  SS%K.J4r4  SS&K5J6r6  SS'K7J8r8  SS(K7J9r9  SS)K:J;r;  \9(       d  SS*K<J=r=  SS+KJ>r>  SS,KJ?r?  SS-KJ@r@  SS.KJArA  \
R                  (       a  \CrDS/rE\8(       a	  S0\E-   S1-   rFOS2rF\+" 5       (       d  \-S3-   \F-   rF " S4 S5\R                  5      rH " S6 S7\R                  5      rJ " S8 S9\R                  5      rLg):$Integration tests for rsync command.    )absolute_import)print_function)division)unicode_literalsN)mock)command)rsync)PopulateProjectId)StorageUrlFromString)	SkipForGS)	SkipForS3)
SkipForXML)"AuthorizeProjectToUseTestingKmsKey)TEST_ENCRYPTION_KEY_S3)TEST_ENCRYPTION_KEY_S3_MD5)BuildErrorRegex)ObjectToURI)ORPHANED_FILE)POSIX_GID_ERROR)POSIX_INSUFFICIENT_ACCESS_ERROR)POSIX_MODE_ERROR)POSIX_UID_ERROR)SequentialAndParallelTransfer)SetBotoConfigForTest)SetEnvironmentForTest)TailSet)unittest)UsingCrcmodExtension)SLOW_CRCMOD_RSYNC_WARNING)ConvertDatetimeToPOSIX)GID_ATTR)	MODE_ATTR)
MTIME_ATTR)NA_TIME)UID_ATTR)Retry)IS_OSX)
IS_WINDOWS)	shim_util)util)DEFAULT_MODE)INVALID_GID)INVALID_UID)USER_IDa_  If you experience problems with multiprocessing on MacOS, they might be related to https://bugs.python.org/issue33725. You can disable multiprocessing by editing your .boto config or by adding the following flag to your command: `-o "GSUtil:parallel_process_count=1"`. Note that multithreading is still available even if you disable multiprocessing.

z"Building synchronization state...
zStarting synchronization...
z>Building synchronization state...
Starting synchronization...

c            	           \ rS rSrSrS r\R                  " S5      \R                  " S5      \R                  " S\R                  " \	R                  R                  S9S9S	 5       5       5       rS
rg)TestRsyncUnit`   z4Unit tests for methods in the commands.rsync module.c                 D   SnSnU R                  [        R                  " [        R                  " U5      5      [        R
                  " U5      5        U R                  [        R                  " [        R
                  " U5      5      [        R                  " U5      5        g )Nu+   gs://bkt/space fslash/plus+tilde~unicodeeèz;gs%3A%2F%2Fbkt%2Fspace+fslash%2Fplus%2Btilde~unicodee%C3%A8)assertEqualr
   
_EncodeUrlsixensure_text
ensure_str
_DecodeUrl)selfdecoded_urlencoded_urls      )platform/gsutil/gslib/tests/test_rsync.pytestUrlEncodeAndDecode$TestRsyncUnit.testUrlEncodeAndDecodec   ss    ?KE  	U%%cook&BC^^K02 	U%%cnn[&AB__[13    zFgslib.utils.copy_helper.TriggerReauthForDestinationProviderIfNecessaryz/gslib.command.Command._GetProcessAndThreadCountzgslib.command.Command.Apply)spec)newc                 V   U R                  5       nU R                  5       nSUl        U R                  SU[	        U5      /5        UR                  [        [	        U5      5      [        R                  SS9  UR                  S S [        R                  R                  R                  SS9  g )N)      r
      )worker_countF)process_countthread_countparallel_operations_overrideprint_macos_warning)CreateTempDirCreateBucketreturn_value
RunCommandsuriassert_called_once_withr   r   ANYr	   CommandParallelOverrideReasonSPEED)r;   !mock_get_process_and_thread_countmock_trigger_reauthpath
bucket_uris        r>   testRsyncTriggersReauth%TestRsyncUnit.testRsyncTriggersReauthp   s     D""$J59%2OOGdD$456//T*-. 0  &==%,__%K%K! > rA    N)__name__
__module____qualname____firstlineno____doc__r?   r   patchMockr	   rT   Applyr[   __static_attributes__r]   rA   r>   r2   r2   `   si    <3 ::NP::?@::+))!6!6799 AP
rA   r2   c                       \ rS rSrS rSrg)TestRsyncUnitWithShim   c                 `   U R                  5       nU R                  5       n[        SS/5         [        SSS.5         U R	                  SSSS	[        U5      U/S
S9nSR                  UR                  S   5      nU R                  SR                  [        R                  " S5      [        U5      U5      U5        SR                  UR                  S   5      nU R                  SU5        U R                  SU5        S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)N)GSUtiluse_gcloud_storageTrue)rk   hidden_shim_modedry_runrm   fake_dir)(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTILCLOUDSDK_ROOT_DIRr
   -apublic-read-PT)return_log_handlerr0   infozpGcloud Storage Command: {} storage rsync --no-ignore-symlinks --predefined-acl publicRead --preserve-posix {} {}warningz'By default, gsutil copies file symlinksz)For preserving POSIX with rsync downloads)rN   rM   r   r   rP   rQ   joinmessagesassertInformatr*   _get_gcloud_binary_path)r;   rZ   fpathmock_log_handler
info_lines
warn_liness         r>   testShimTranslatesFlags-TestRsyncUnitWithShim.testShimTranslatesFlags   s,   ""$J E	GHJ 
K 6<)" 	  ??dM4:&/# + % YY/88@A
&&,f11*=tJ?O'"	$ YY/88CD
?LA:N#	
K 
K	 	
K 
Ks#   DC D=D
D	D
D-r]   N)r^   r_   r`   ra   r   rf   r]   rA   r>   rh   rh      s    OrA   rh   c                   x   \ rS rSrSrS rS r SKS jrS rS r	\
R                  " \S5      \
R                  " \" 5       S	5      S
 5       5       rS rS rS rS rS r\\
R                  " \" 5       S	5      S 5       5       r\\
R                  " \" 5       S	5      S 5       5       r\\
R                  " \" 5       S	5      S 5       5       r\
R                  " \" 5       S	5      S 5       r\
R                  " \" 5       S	5      S 5       rS r\
R                  " \" 5       S	5      S 5       r\\
R                  " \" 5       S	5      S 5       5       r\
R                  " \S5      S 5       r\\
R                  " \S5      \
R                  " \" 5       S	5      S 5       5       5       r\
R                  " \" 5       S	5      S 5       r \
R                  " \" 5       S	5      S 5       r!\
R                  " \" 5       S	5      \"" S5      S 5       5       r#\
R                  " \S5      S 5       r$\
R                  " \S5      S  5       r%\"" S!5      S" 5       r&S# r'S$ r(S% r)S& r*S' r+S( r,S) r-S* r.S+ r/S, r0S- r1S. r2S/ r3S0 r4S1 r5S2 r6S3 r7S4 r8\
R                  " \S55      S6 5       r9\
R                  " \S75      S8 5       r:S9 r;S: r<S; r=S< r>\"" S=5      \?" S>5      \S? 5       5       5       r@\"" S=5      \?" S>5      \S@ 5       5       5       rA\"" S=5      \?" S>5      \SA 5       5       5       rB\"" SB5      SC 5       rC\D" SD5      SE 5       rESF rF\D" SG5      SH 5       rGSIrHgJ)L	TestRsync   r   c                     U R                   S:X  a  U R                  OU R                  nUR                  UUU R                   U/S9n[	        XSS5      $ )aI  Retrieves and returns an attribute from an objects metadata.

Args:
  bucket_name: The name of the bucket the object is in.
  object_name: The name of the object itself.
  attr_name: The name of the custom metadata attribute.

Returns:
  The value at the specified attribute name in the metadata. If not present,
  returns None.
gs)providerfieldsN)default_providerjson_apixml_apiGetObjectMetadatagetattr)r;   bucket_nameobject_name	attr_name
gsutil_apimetadatas         r>   _GetMetadataAttributeTestRsync._GetMetadataAttribute   s_     **d2 --8< ++K,7595J5J4=; , @H 8--rA   c                 x    U R                   (       a  U R                  SU5        g U R                  [        U5        g )N	Completed)_use_gcloud_storageassertNotInr5   
NO_CHANGES)r;   stderrs     r>   _VerifyNoChangesTestRsync._VerifyNoChanges   s+    
{F+
z6*rA   c                 2    U R                  UU[        UUS9  g)aB  Retrieves the object's mtime.

Args:
  bucket_name: The name of the bucket the object is in.
  object_name: The name of the object itself.
  expected_mtime: The expected retrieved mtime.
  expected_present: True if the mtime must be present in the
      object metadata, False if it must not be present.


Returns:
  None
expected_presentN)VerifyObjectCustomAttributer$   )r;   r   r   expected_mtimer   s        r>   _VerifyObjectMtimeTestRsync._VerifyObjectMtime   s&    $ 	$$[%0%/%36F	 % HrA   c                 2   U R                  5       nU R                  USSS9nU R                  5       nU R                  S[	        U5      [	        U5      /SS9  U R                  S[	        U5      [	        U5      /SS9  U R                  S[	        U5      SU R
                  -   /SS9  U R                  S[	        U5      U/SS9  U R                  (       d  U R                  SU[	        U5      /SS9  U R                  SU[	        U5      SU R
                  -   /SS9  g)	z%Tests various invalid argument cases.obj1   obj1rZ   r   contentsr
      )expected_statuszgs://N)rN   CreateObjectrM   	RunGsUtilrQ   nonexistent_bucket_namer   )r;   rZ   r   tmpdirs       r>   test_invalid_argsTestRsync.test_invalid_args   s$   ""$J
)/&-  /D !FNNGT$Zj)9:ANNNNGT*-tDz:ANNNN		j	7T%A%AA	C  
 	NNGT$Z0!ND## nngvtDz2AnFNN	&	dWt;;;	=  rA   c                 R  ^ ^^ T R                  5       mT R                  5       mT R                  TSSSS9  T R                  TSSSS9  T R                  TSS	[        S
5      S9  T R                  TSSSS9  T R                  TSSSS9  [        [        SSS9UUU 4S j5       nU" 5         g)zDTests that an exception is thrown if mtime cannot be cast as a long.r   r   xyzrZ   r   r   mtimeobj2   obj2{   obj3   obj3l   Cc}obj4   obj4iobj5   obj5rF   r   triestimeout_secsc                    > TR                  S[        T5      [        T5      /SS9n TR                  (       a[  TR                  U S5        TR	                  U S5        TR                  U S5        TR                  U S5        TR                  U S5        g TR                  S	U 5        TR                  S
U 5        TR                  SU 5        TR                  SU 5        TR                  SU 5        g )Nr
   Treturn_stderrzNobj1#\d+ metadata did not contain a numeric value for goog-reserved-file-mtimezNobj2#\d+ metadata did not contain a numeric value for goog-reserved-file-mtimezNobj3#\d+ metadata that is more than one day in the future from the system timez)Found negative time value in gs://.*/obj4z)Found negative time value in gs://.*/obj5z)obj1 has an invalid mtime in its metadataz)obj2 has an invalid mtime in its metadataz:obj3 has an mtime more than 1 day from current system timez)obj4 has a negative mtime in its metadataz)obj5 has a negative mtime in its metadata)r   rQ   r   assertRegexassertNotRegexr{   r   r   bucket1_uribucket2_urir;   s    r>   _Check11TestRsync.test_invalid_src_mtime.<locals>._Check1'  s    ~~D%-1  3f 
	!	! )	* 	 )	* 	 	! 	!MN!MNA6JDfM	 	A6JA6JrA   N)rN   r   longr'   AssertionError)r;   r   r   r   s   ` @@r>   test_invalid_src_mtime TestRsync.test_invalid_src_mtime  s     ##%K##%K"(&!  # 	"(&  ! 	"(& /  1
 	"(&   " 	"(&    >3K 4K4 IrA   z*POSIX attributes not available on Windows.zTest requires fast crcmod.c           
      T  ^ ^^ T R                  5       mT R                  5       m[        R                  " 5       n[        R                  " 5       nT R                  TSSSS9  T R                  TSSUS9  T R                  TSS	US9  T R                  TS
S	[        5       [        5       SS9  T R                  TSS[        U[        [        5      S9  T R                  TSSS9  [        [        SSS9UU U4S j5       nU" 5         T R                  TR                  S[        S5        T R                  TR                  S[         [        U5      5        T R                  TR                  S[         [        U5      5        T R                  TR                  S
[         [        [        5       5      5        T R                  TR                  S
["        [        [        5       5      5        T R                  TR                  S
[        S5        T R                  TR                  S["        [        [        5      5        T R                  TR                  S[         [        U5      5        T R                  TR                  S[        [        [        5      5        [        [        SSS9UU U4S j5       nU" 5         g)z0Tests that rsync -P works with bucket to bucket.r   r   444rZ   r   r   moder   r   rZ   r   r   gidr   r   r   222rZ   r   r   uidr   r   r   r   r   rF   r   r   c                    > TR                  SS[        T5      [        T5      /SS9n [        [        T5      TR                  T5      5      n[        [        T5      TR                  T5      5      nTR	                  U[        / SQ5      5        TR	                  U[        / SQ5      5        TR                  (       a  TR                  SU 5        gTR                  SU 5        g)	z?Test bucket to bucket rsync with -P flag and verify attributes.r
   ru   Tr   )/obj1/obj2/obj3/obj4z/obj5Patching,Copying POSIX attributes from src to dst forN)r   rQ   r   FlatListBucketr5   setr   r{   )r   listing1listing2
dst_bucketr;   
src_buckets      r>   r   ?TestRsync.test_bucket_to_bucket_preserve_posix.<locals>._Check1r  s     ~~D$z*
  f j)4+>+>z+JKhj)4+>+>z+JKh
xHIK xHIK		!	!j&)DfMrA   c                     > TR                  SS[        T5      [        T5      /SS9n TR                  (       a  TR                  SU 5        gTR                  SU 5        g)zBCheck that we are not patching destination metadata a second time.r
   ru   Tr   r   r   N)r   rQ   r   r   r   r   r;   r   s    r>   _Check2?TestRsync.test_bucket_to_bucket_preserve_posix.<locals>._Check2  s`     ~~D$z*
  f 
	!	!V,GPrA   N)rN   osgetgidr+   GetNonPrimaryGidr   r.   r-   r/   strr,   r'   r   r   r   r#   r"   r&   )r;   primary_gidnon_primary_gidr   r   r   r   s   `    @@r>   $test_bucket_to_bucket_preserve_posix.TestRsync.test_bucket_to_bucket_preserve_posixD  s    ""$J""$J))+K++-O"(&   " 	"(&%  ' 	"(&)  + 	"(&%-%-   " 	"(&!%|,  . 	"(&  (
 >3N 4N& I$$Z%;%;VY%*,$$Z%;%;VX%(%57$$Z%;%;VX%(%9;
 	$$Z%;%;VX%(%79$$Z%;%;VX%(%79$$Z%;%;VY%*, 	$$Z%;%;VX%(\3$$Z%;%;VX%(%57$$Z%;%;VY%(%68 >3	Q 4	Q IrA   c                 r  ^ ^^ T R                  5       mT R                  5       mT R                  TSSSS9  T R                  TSSSS9  T R                  TSSS9  T R                  TSSS9  [        [        S	SS
9UU U4S j5       nU" 5         [        [        S	SS
9UU U4S j5       nU" 5         T R                  (       aA  T R                  TR                  S[        SS9  T R                  TR                  S[        SS9  gT R                  TR                  SS5        T R                  TR                  SS5        g)zTests bucket to bucket with mtime.

Each has the same items but only the source has mtime stored in its
metadata.
Ensure that destination now also has the mtime of the files in its metadata.
r   r   r   r   subdir/obj2   subdir/obj2r   r   rF   r   c                     > TR                  SS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n TR	                  U [        SS/5      5        gTests rsync works as expected.r
   -rr   /subdir/obj2Nr   rQ   r   r   r5   r   )r   r   r;   r   s    r>   r   GTestRsync.test_bucket_to_bucket_same_objects_src_mtime.<locals>._Check1  sZ     nngtT*%5tJ7GHIj)4+>+>z+JKh
xg~%>!?@rA   c                  r   > TR                  S[        T5      [        T5      /SS9n TR                  U 5        g Nr
   Tr   r   rQ   r   r   s    r>   r   GTestRsync.test_bucket_to_bucket_same_objects_src_mtime.<locals>._Check2  s@    ~~D$
,0  2f F#rA   Fr   01N)rN   r   r'   r   r   r   r   r%   )r;   r   r   r   r   s   `  @@r>   ,test_bucket_to_bucket_same_objects_src_mtime6TestRsync.test_bucket_to_bucket_same_objects_src_mtime  sj    ""$J""$J"(&   	"/-   	"(&  ( 	"/-  /
 >3A 4A I >3$ 4$ I j44$%/4  6 j44+%/4  6 j44fcB
j44mSIrA   c                   ^ ^^ T R                  5       mT R                  5       mT R                  TSSSS9nT R                  TSSSS9nT R                  UR                  UR                  S5        T R                  UR                  UR                  S	5        [        [        S
SS9UU U4S j5       nU" 5         T R                  TR                  SS5        T R                  TR                  SS	5        g)z7Tests bucket to bucket where source has mtime in files.r   r   r   r   r   r   r   r  r  rF   r   c                  X  > TR                  SS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        SS/5      5        TR	                  U[        SS/5      5        gr   r   r   r   r   r;   r   s     r>   r   :TestRsync.test_bucket_to_bucket_src_mtime.<locals>._Check1  s     nngtT*%5tJ7GHIj)4+>+>z+JKhj)4+>+>z+JKh
xg~%>!?@
xg~%>!?@rA   N)rN   r   r   r   r   r'   r   )r;   r   r   r   r   r   s   `   @@r>   test_bucket_to_bucket_src_mtime)TestRsync.test_bucket_to_bucket_src_mtime  s     ""$J""$J
)/&-#$  &D 
)6&4#$  &D
 	D,,d.>.>DD,,d.>.>D >3A 4A I 	J22FC@J22M3GrA   c                   ^ ^	^
 T R                  5       m
T R                  5       m	T R                  T
SSS9  T R                  T
SSS9  T R                  T
SSS9  T R                  T
SS	S9  T R                  T
S
SSS9  T R                  T	SSSS9  T R                  T	SSSS9  T R                  T	SS[        S5      S9  T R                  T	SSSS9  T R                  T	S
SSS9  [        [        SSS9U	U U
4S j5       nU" 5         T R                  T	R                  SS5        T R                  (       a  T R                  T
R                  SS/5      R                  n[        [        U5      5      nT R                  T	R                  SU5        T R                  T
R                  SS/5      R                  n[        [        U5      5      nT R                  T	R                  SU5        O@T R                  T	R                  S[        SS9  T R                  T	R                  S[        SS9  [        [        SSS9U	U U
4S j5       nU" 5         [        [        SSS9U	U 4S j5       nU" 5         [        [        SSS9U	U U
4S j5       nU" 5         g )!z>Tests bucket to bucket where destination has mtime in objects.r      OBJ1r   r   r   .obj3s   .obj3zsubdir/obj4s   subdir/obj4obj6   OBJ6d   r   r   
   s   .OBJ3l    J)subdir/obj5   subdir/obj5   obj6rF   r   r   c            	      Z  > TR                  SSS[        T5      [        T5      /SS94  [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        / SQ5      5        TR	                  U[        / SQ5      5        g)r   r
   r   -dTr   r   r   z/.obj3z/subdir/obj4/obj6Nr   r	  s     r>   r   :TestRsync.test_bucket_to_bucket_dst_mtime.<locals>._Check1A  s     nngtT:&:&( $(  ) *
 j)4+>+>z+JKhj)4+>+>z+JKh


J
KM 

J
KMrA   10timeCreatedFr   c                  r   > TR                  S[        T5      [        T5      /SS9n TR                  U 5        g r   r  r   s    r>   _Check3:TestRsync.test_bucket_to_bucket_dst_mtime.<locals>._Check3r  sB     ~~D$
,0  2f F#rA   c            
        > TR                  STR                  S[        T S5      /SS95        TR                  STR                  S[        T S5      /SS95        TR                  STR                  S[        T S5      /SS95        g )	NOBJ1catr   Treturn_stdoutr  OBJ6r  )r5   r   rQ   )r   r;   s   r>   _Check4:TestRsync.test_bucket_to_bucket_dst_mtime.<locals>._Check4}  s     

..%j&!9:$.
OQ
 

..%j'!:;'+  -. 

..%j&!9:$.
OQrA   c            
        > TR                  SSSS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        / SQ5      5        TR	                  U[        / SQ5      5        TR	                  STR                  S[        TS5      /S	S
95        TR                  TR                  SS5        g)!Tests rsync -c works as expected.r
   r   r  -cr  r&  r#  r  Tr$  100Nr   rQ   r   r   r5   r   r   r   r	  s     r>   _Check5:TestRsync.test_bucket_to_bucket_dst_mtime.<locals>._Check5  s     nnD$

 j)4+>+>z+JKhj)4+>+>z+JKh


J
KM 

J
KM
 

..%j&!9:$.
OQ j44feDrA   N)rN   r   r   r'   r   r   r   r   GetObjectMetadataWithFieldsr  r   r!   r%   )r;   r   source_o1_time_createdsource_o1_posix_time_createdsource_o4_time_createdsource_o4_posix_time_createdr  r'  r.  r   r   s   `        @@r>   test_bucket_to_bucket_dst_mtime)TestRsync.test_bucket_to_bucket_dst_mtime  s    ""$J""$J"(&  ( 	"/-  / 	")'  ) 	"/-  / 	"(&  ! 	"(&    	"/-    	")' /  1 	"/-    	"(&  ! >3M 4M$ I 	J22M4H  $??

 
 &=/ ;;F; %(
 !7
8&:"
j44f:<#??

 
 -- BBM+ %(
 !7
8&:"
j44m:< j44$%/4  6 j44+%/4  6 >3$ 4$ I >3Q 4Q$ I >3E 4E0 IrA   c                    ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSSS9  T R                  TSS	S9  T R                  TS
SSS9  T R                  TSSS9  T R                  TSSS9  T R                  TSSS9  T R                  TS
SSS9  [        [        SSS9UUU 4S j5       nU" 5         [        [        SSS9UUU 4S j5       nU" 5         T R                  TS
SS9  T R                  TSSS9  T R	                  S[        TS5      /5        T R	                  S[        TS5      /5        [        [        SSS9UUU 4S j5       nU" 5         [        [        SSS9UUU 4S j5       nU" 5         g)FTests that flat and recursive rsync between 2 buckets works correctly.r   r   r   .obj2   .obj2r  r   subdir/obj3   subdir/obj3r  s   obj6_r     .OBJ2r   r   r  r  r  rF   r   r   c            
        > TR                  S[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        / SQ5      5        TR	                  U[        / SQ5      5        TR	                  STR                  S[        TS5      /SS95        TR	                  STR                  S[        TS5      /SS95        TR	                  STR                  S[        TS	5      /SS95        TR                  TR                  SS
5        g)r   r
   )r   /.obj2/subdir/obj3r  )r   r?  r   /subdir/obj5r  r9  r#  Tr$  obj6_r  r  Nr-  r   r   r   r   r;   s     r>   r   0TestRsync.test_bucket_to_bucket.<locals>._Check1  sI    nngtK0${2CDEk*D,?,?,LMhk*D,?,?,LMh
xGHJ
 
CMNP
 

..%k7!;<'+  -. 

..%k7!;<'+  -. 

..%k6!:;'+  -.
 k55wErA   c                  r   > TR                  S[        T5      [        T5      /SS9n TR                  U 5        g r   r  r   s    r>   r   0TestRsync.test_bucket_to_bucket.<locals>._Check2  sB     ~~D%-1  3f F#rA   obj7   obj7rmc                  X  > TR                  SS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        / SQ5      5        TR	                  U[        / SQ5      5        g )Nr
   r   r?  r  r@  )r   r?  r   r  /obj7r@  rA  r   rC  s     r>   r  0TestRsync.test_bucket_to_bucket.<locals>._Check3  s    
nngtT+%6[8IJKk*D,?,?,LMhk*D,?,?,LMh
x%H!IJ 

  rA   c                  t   > TR                  SS[        T5      [        T5      /SS9n TR                  U 5        g )Nr
   r   Tr   r  r   s    r>   r'  0TestRsync.test_bucket_to_bucket.<locals>._Check4#  D     ~~D${+  f F#rA   NrN   r   r'   r   r   rQ   r;   r   r   r  r'  r   r   s   `    @@r>   test_bucket_to_bucketTestRsync.test_bucket_to_bucket  s    ##%K##%K"(&  ( 	")'    	"/-  / 	"('  !
 	")'  ) 	"(&  ( 	"/-  / 	"(&  ! >3F 4F> I >3$ 4$ I 	"(&  ( 	"(&  ( 	NND${F345NND${G456 >3 4  I >3$ 4$ IrA   c                   ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  TSSS9  T R                  TSSS9  T R                  TS	S
S9  T R                  TSSS9  [        [        SSS9UUU 4S j5       nU" 5         [        [        SSS9UUU 4S j5       nU" 5         T R                  TSSS9  T R                  TSSS9  T R	                  S[        TS5      /5        T R	                  S[        TS5      /5        [        [        SSS9UUU 4S j5       nU" 5         [        [        SSS9UUU 4S j5       nU" 5         g)r8  r   r   r   r9  r:  r;  r<  r=  r   r   r  r  rF   r   r   c            
        > TR                  SS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        / SQ5      5        TR	                  U[        / SQ5      5        TR	                  STR                  S[        TS5      /SS95        TR	                  STR                  S[        TS5      /SS95        g	)
r   r
   r  r   r?  r@  r   r?  rA  r9  r#  Tr$  Nr   rC  s     r>   r   8TestRsync.test_bucket_to_bucket_minus_d.<locals>._Check1K  s     nngtT+%6[8IJKk*D,?,?,LMhk*D,?,?,LMh
x%H!IJ x%H!IJ 

..%k7!;<'+  -. 

..%k7!;<'+  -.rA   c                  t   > TR                  SS[        T5      [        T5      /SS9n TR                  U 5        g Nr
   r  Tr   r  r   s    r>   r   8TestRsync.test_bucket_to_bucket_minus_d.<locals>._Check2e  rP  rA   r  r  rG  rH  rI  c            	      Z  > TR                  SSS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        / SQ5      5        TR	                  U[        / SQ5      5        g Nr
   r  r   rK  r   rC  s     r>   r  8TestRsync.test_bucket_to_bucket_minus_d.<locals>._Check3{  s    
nnD$ k*D,?,?,LMhk*D,?,?,LMh
x%H!IJ x%H!IJrA   c            	      v   > TR                  SSS[        T5      [        T5      /SS9n TR                  U 5        g Nr
   r  r   Tr   r  r   s    r>   r'  8TestRsync.test_bucket_to_bucket_minus_d.<locals>._Check4  I     ~~D$ 	  f
 F#rA   NrQ  rR  s   `    @@r>   test_bucket_to_bucket_minus_d'TestRsync.test_bucket_to_bucket_minus_d.  s    ##%K##%K"(&  ( 	")'  ) 	"/-  / 	")'  ) 	"(&  ( 	"/-  /
 >3. 4., I >3$ 4$ I 	"(&  ( 	"(&  ( 	NND${F345NND${G456 >3K 4K I >3$ 4$ IrA   c                 :  ^ ^^^ T R                  5       m[        R                  R                  TS5      n[        R                  " U5        T R                  TSSSS9  T R                  TSSSS9  T R                  USS	SS9  T R                  US
SSS9  T R                  TSSSS9  T R                  TSSSS9  T R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  T R                  TR                  SSS5        T R                  TSSS9  T R                  TSSSS9  T R                  TSSSS9  T R                  TSSSS9  [        5       m[        [        SSS9UUU U4S j5       nU" 5         [        [        SSS9UU U4S j5       nU" 5         T R                  TR                  SS 5        T R                  TR                  SS 5        T R                  TR                  S!S 5        T R                  TR                  SS"5        T R                  TR                  SS#5        S$[        TS5      -  nUT;  a/  T R                   (       d  T R#                  TR                  SSS5        [        [        SSS9UU U4S% j5       nU" 5         g&)'a7  Tests dir to bucket with mtime.

Each has the same items, the source has mtime for all objects, whereas dst
only has mtime for obj5 and obj6 to test for different a later mtime at src
and the same mtime from src to dst, respectively. Ensure that destination
now also has the mtime of the files in its metadata.
subdirr   r   r  r   	file_namer   r   r9  r:  r   r<  r   r     r  r  r  rG     obj7_r  r   testr   r   r  r   r  rH  rF   r   r   c            
      ^  > TR                  SSST[        T5      /SS9n TR                  U R                  5        Vs/ s H  o(       d  M  UPM     sn5        [	        TTR                  T5      5      n[	        [        T5      TR                  T5      5      nTR                  U[        / SQ5      5        TR                  U[        / SQ5      5        TR                  STR                  S[        TS	5      /SS
95        TR                  STR                  S[        TS5      /SS
95        gs  snf )r   r
   r   r  Tr   r   r?  r@  rA  r  rL  r&  r#  r  r$  obj7_rG  N)	r   rQ   update
splitlinesr   FlatListDirr   r5   r   )r   sr   r   rZ   cumulative_stderrr;   r   s       r>   r   3TestRsync.test_dir_to_bucket_mtime.<locals>._Check1  s3    ~~wdF#J/1,0  2f 6+<+<+>D+>a!+>DE!1!1&!9:hj)4+>+>z+JKh


   

   

..%j&!9:$.
OQ 

..%j&!9:$.
OQ/  Es   
D*D*c            	      d   > TR                  SSST[        T5      /SS9n TR                  U 5        g )Nr
   r   r  Tr   r  r   rZ   r;   r   s    r>   r   3TestRsync.test_dir_to_bucket_mtime.<locals>._Check2   @     ~~wdF#J/1,0  2f F#rA   r  r;  15r,  zkCopying whole file/object for %s instead of patching because you don't have owner permission on the object.c            
        > TR                  SSSST[        T5      /5        [        TTR                  T5      5      n [        [        T5      TR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        TR                  STR                  S[        TS5      /SS	95        TR                  TR                  SS
5        g)r*  r
   r   r  r+  rn  r  r#  Tr$  r,  N)	r   rQ   r   rr  r   r5   r   r   r   r   r   rZ   r;   r   s     r>   r'  3TestRsync.test_dir_to_bucket_mtime.<locals>._Check4  s     nngtT4j9IJK!1!1&!9:hj)4+>+>z+JKh


   

   

..%j&!9:$.
OQ j44feDrA   N)rM   r   rY   ry   mkdirCreateTempFilerN   r   !_SetObjectCustomMetadataAttributer   r   r   r'   r   r   rQ   r   r   )	r;   rg  r   r   copied_over_object_noticer'  rZ   rt  r   s	   `     @@@r>   test_dir_to_bucket_mtime"TestRsync.test_dir_to_bucket_mtime  s    !FWW\\&(+FHHVv"(!(   " 	v")!)   " 	v"(!/   " 	v"(!/   " 	v"(!(!  # 	v"(!)!  # ""$J"(&  ( 	")'  ) 	**4+@+@+5+A+A7+16; 	"(&  ( 	"/-    	"(&  ! 	"(&  !
 
>3Q 4Q@ I >3$ 4$ I 	J22FDAJ22GTBJ22M4HJ22M4HJ22FEB	6Z!	"  	")::$$ &&z'='=w'-/
 >3E 4E4 IrA   c                 2  ^  [        [        SSS9U 4S j5       nU" 5         T R                  SS9nT R                  5       n[	        SS/5         T R                  SS	S
SU[        U5      /SS9nT R                  SU5        SSS5        g! , (       d  f       g= f)z5Tests that rsync seek-ahead iterator works correctly.rF   r   r   c                    > TR                  5       n [        R                  R                  U S5      n[        R                  " U5        TR                  U SSS9  TR                  U SSS9  TR                  USSS9  TR                  5       nTR                  USS	S
9  TR                  USSS
9  TR                  USSS
9  TR                  US5        [        SS/5         TR                  SSSSU [        U5      /SS9nTR                  SU5        TR                  US5        TR                  SSSSU [        U5      /SS9nTR                  SU5        SSS5        g! , (       d  f       g= f)z*Test estimating an rsync upload operation.rg  r   r   r   ri  r   r9  r:  r   r<  r=  r   r   r   r  r  rF   )rk   task_estimation_thresholdr  rk   task_estimation_forcerm   -mr
   r  r   Tr   z;Estimated work for this command: objects: 5, total size: 20Estimated workN)rM   r   rY   ry   r~  r  rN   r   AssertNObjectsInBucketr   r   rQ   r{   r   )r   rg  rZ   r   r;   s       r>   r   8TestRsync.test_dir_to_bucket_seek_ahead.<locals>._Check1B  s    !!#fww||FH-fhhv
6GL
7XN
$*#1  3 $$&j
:$+!)  + :$*!(  * :$1!/  1
 !!*a0!M!L!N O7D$*     	I	 	##J27D$*     	)62%O O Os   A5E
E)
test_files)rk   r  r  r  r  r
   r  r   Tr   r  N)r'   r   rM   rN   r   r   rQ   r   )r;   r   r   rZ   r   s   `    r>   test_dir_to_bucket_seek_ahead'TestRsync.test_dir_to_bucket_seek_ahead<  s     >3*3 4*3X I1-F""$J 
KJL 
M~~$f
  f '0
M 
M 
Ms   2B
Bc                   ^ ^^	^
 T R                  5       m
[        R                  R                  T
S5      n[        R                  " U5        T R                  5       mT R                  T
SSS9  T R                  T
SSS9m	T R                  USSS9  T R                  TSS	S
9  T R                  TSSS
9  T R                  TSSS
9  T R                  TS5        [        [        SSS9UU	U U
4S j5       nU" 5         [        [        SSS9UU U
4S j5       nU" 5         [        [        SSS9UU U
4S j5       nU" 5         [        [        SSS9UU U
4S j5       nU" 5         T R                  T
SSS9  T R                  TSSS
9  [        R                  " [        R                  R                  T
S5      5        T R                  S[        TS5      /5        [        [        SSS9UU U
4S j5       nU" 5         [        [        SSS9UU U
4S j5       nU" 5         g)zBTests that flat and recursive rsync dir to bucket works correctly.rg  r   r   r  r9  r:  r   r<  r=  r   r   r   r  r  rF   r   r   c                  r  > TR                  SST	[        T5      /5        [        T	TR                  T	5      5      n [        [        T5      TR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        [        [        R                  R                  T	S5      5       nTR                  SSR                  UR                  5       5      5        SSS5        TR                  S[        TS5      /S	S
9nTR                  (       an  [        [        R                  R                  T5      5      n[        TR!                  TR"                  SS5      5      nTR%                  US:H  =(       d    XE:H  5        gTR                  SU5        g! , (       d  f       N= f)r   r
   r  rW  rX  r9  r0   Nr#  Tr$  r  )r   rQ   r   rr  r   r5   r   openr   rY   ry   	readlinesr   intgetmtimer!   r   r   
assertTrue)
r   r   fcloud_obj2_contentlocal_obj2_mtimecloud_obj2_ctimerZ   o2_pathr;   r   s
         r>   r   5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check1  sm    nngtVT*-=>?!1!1&!9:hj)4+>+>z+JKh
x%H!IJ x%H!IJ VW-.!$))AKKM":; />>$z7+
,D * B		!	! rww//891&&z'='=w'467 	*g5 =(<	> 	"45 /.s   0F((
F6c                  b   > TR                  SST[        T5      /SS9n TR                  U 5        g r[  r  rw  s    r>   r   5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check2  :     ~~D&$z"2
34  If
F#rA   c            
      r  > TR                  SSST[        T5      /5        [        TTR                  T5      5      n [        [        T5      TR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        [        [        R                  R                  TS5      5       nTR                  SSR                  UR                  5       5      5        SSS5        TR                  STR                  S	[        TS5      /S
S95        g! , (       d  f       N;= f)r*  r
   r  r+  rW  rX  r9  r0   Nr#  Tr$  )r   rQ   r   rr  r   r5   r   r  r   rY   ry   r  r   r   r  rZ   r;   r   s      r>   r  5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check3  s     nngtT64
3CDE!1!1&!9:hj)4+>+>z+JKh
x%H!IJ x%H!IJ VW-.!$))AKKM":; /


..%j'!:;'+  -. /.s   0D((
D6c            	      d   > TR                  SSST[        T5      /SS9n TR                  U 5        g Nr
   r  r+  Tr   r  rw  s    r>   r'  5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check4  ry  rA   r  r  rG  rH  rI  c            	      6  > TR                  SSST[        T5      /5        [        TTR                  T5      5      n [        [        T5      TR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        g r^  r   rQ   r   rr  r   r5   r   r|  s     r>   r.  5TestRsync.test_dir_to_bucket_minus_d.<locals>._Check5  s    
nngtT64
3CDE!1!1&!9:hj)4+>+>z+JKh
x%H!IJ x%H!IJrA   c            	      d   > TR                  SSST[        T5      /SS9n TR                  U 5        g ra  r  rw  s    r>   _Check65TestRsync.test_dir_to_bucket_minus_d.<locals>._Check6  ry  rA   N)rM   r   rY   ry   r~  rN   r  r   r  r'   r   unlinkr   rQ   )r;   rg  r   r   r  r'  r.  r  rZ   r  r   s   `       @@@r>   test_dir_to_bucket_minus_d$TestRsync.test_dir_to_bucket_minus_d~  s:    !FWW\\&(+FHHV""$Jv'J!!,3+3 " 5G 	v"(!/  1 	")'  ) 	"(&  ( 	"/-  / 	
A. >36 46< I >3$ 4$ I >3. 4.& I >3$ 4$ I 	v'J"(&  ( IIbggll66*+NND$z7345 >3K 4K I >3$ 4$ IrA   c                 V  ^ ^	^
 T R                  5       m	T R                  5       m
[        R                  R                  T	S5      n[        R                  R                  T
S5      n[        R                  " U5        [        R                  " U5        T R                  T	SSSS9  T R                  T	SSSS9  T R                  US	S
SS9  T R                  T	SSSS9  T R                  T	SSSS9  T R                  T
SSSS9  T R                  T
SSSS9  T R                  USSSS9  T R                  T
SSSS9  T R                  T
SSSS9  T R                  SSST	T
/5        [        T	T R                  T	5      5      n[        T
T R                  T
5      5      nT R                  U[        / SQ5      5        T R                  U[        / SQ5      5        [        [        R                  R                  T
S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  T
S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  T
S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        U U	U
4S  jnU" 5         T R                  SSSS!T	T
/5        [        T	T R                  T	5      5      n[        T
T R                  T
5      5      nT R                  U[        / SQ5      5        T R                  U[        / SQ5      5        [        [        R                  R                  T	S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  T	S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  T
S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        U U	U
4S" jnU" 5         [        R                  " [        R                  R                  T	S5      5        [        R                  " [        R                  R                  T
S5      5        T R                  T	SSSS9  T R                  T
SSSS9  [        R                  " [        R                  R                  T	S5      5        [        R                  " [        R                  R                  T
S5      5        T R                  SSST	T
/5        [        T	T R                  T	5      5      n[        T
T R                  T
5      5      nT R                  U[        / S#Q5      5        T R                  U[        / S#Q5      5        U U	U
4S$ jnU" 5         g! , (       d  f       GN#= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNQ= f! , (       d  f       GN= f)%?Tests that flat and recursive rsync dir to dir works correctly.subdir1subdir2r   r   r  rh  r9  r:  r      subdir1/obj3r  r  r  rG  rk  r=    r   r   r      subdir2/obj5r  rH  r
   r   r  )r   r?  /subdir1/obj3r  rL  r0   Nr&  ro  c            	      V   > T R                  [        T R                  SSTT/SS95        g r[  r5   r   r   r;   tmpdir1tmpdir2s   r>   r   0TestRsync.test_dir_to_dir_mtime.<locals>._Check1Y  .    


..'4':$.
OQrA   r+  c            
      X   > T R                  [        T R                  SSSTT/SS95        g r  r  r  s   r>   r   0TestRsync.test_dir_to_dir_mtime.<locals>._Check2t  5    


..'4w@'+  -.rA   r?  r  r  c            
      X   > T R                  [        T R                  SSSTT/SS95        g ra  r  r  s   r>   r  0TestRsync.test_dir_to_dir_mtime.<locals>._Check3  r  rA   rM   r   rY   ry   r~  r  r   r   rr  r5   r   r  r  r  )r;   r  r  r   r   r  r   r   r  r  r  s   `        @@r>   test_dir_to_dir_mtimeTestRsync.test_dir_to_dir_mtime  s      "G  "Gggll7I.Gggll7I.GHHWHHWw"(!(   " 	w")!)   " 	w"(!0   " 	w"(!(!  # 	w"(!)!  # 	w")!)"  $ 	w"(!(   " 	w"(!0   " 	w"(!(!  # 	w"(!(!  #
 	NNGT4':;w 0 0 9:Hw 0 0 9:H#LMO 	#LMO 
bggll7G,	-
w		!++- 89 
.	bggll7F+	,
vtyy78 
-	bggll7F+	,
w		!++- 89 
-Q I 	NNGT4w@Aw 0 0 9:Hw 0 0 9:H#LMO 	#LMO 
bggll7G,	-
w		!++- 89 
.	bggll7G,	-
w		!++- 89 
.	bggll7F+	,
vtyy78 
-. I IIbggll7F+,IIbggll7F+,w"(!(   " 	w"(!(!  # IIbggll7F+,IIbggll7G,-NNGT4':;w 0 0 9:Hw 0 0 9:HXs#GHIXs#GHI. IQ 
.	-	,	,	,	,. 
.	-	-	-	,	,sH   30X?0Y50Y#,0Y50Z.0Z?
Y
Y #
Y25
Z
Z
Z(c                   ^ ^
^ T R                  5       m
T R                  5       m[        R                  R                  T
S5      n[        R                  R                  TS5      n[        R                  " U5        [        R                  " U5        T R                  T
SSS9  T R                  T
SSS9  T R                  USS	S9  T R                  TSS
S9  T R                  TSSS9  T R                  USSS9  T R                  SST
T/5        [        T
T R                  T
5      5      n[        TT R                  T5      5      nT R                  U[        / SQ5      5        T R                  U[        / SQ5      5        [        [        R                  R                  T
S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  TS5      5       nT R                  SSR                  UR                  5       5      5        SSS5        U U
U4S jnU" 5         T R                  SSST
T/5        [        T
T R                  T
5      5      n[        TT R                  T5      5      nT R                  U[        / SQ5      5        T R                  U[        / SQ5      5        [        [        R                  R                  T
S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  T
S5      5       nT R                  SSR                  UR                  5       5      5        SSS5        U U
U4S jnU" 5         T R                  T
SSS9  T R                  TSSS9  [        R                  " [        R                  R                  T
S5      5        [        R                  " [        R                  R                  TS5      5        T R                  SSST
T/5        [        T
T R                  T
5      5      n[        TT R                  T5      5      nT R                  U[        / SQ5      5        T R                  U[        / SQ5      5        U U
U4S jnU" 5         T R                  5       m
T R                  5       mT R                  T
SSS9  T R                  T
S S!S9  T R                  TS S!S9  T R                  SST
T/5        [        T
T R                  T
5      5      n[        TT R                  T5      5      nT R                  U[        S"S#/5      5        T R                  U[        S"S#/5      5        U U
U4S$ jn	U	" 5         g! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GN[= f)%r  r  r  r   r   r  r9  r:  r   r  r=  r   r   r   r  r
   r  )r   r?  r  )r   r?  z/subdir2/obj5r0   Nz.OBJ2c            	      V   > T R                  [        T R                  SSTT/SS95        g r[  r  r  s   r>   r   2TestRsync.test_dir_to_dir_minus_d.<locals>._Check1  r  rA   r+  c            
      X   > T R                  [        T R                  SSSTT/SS95        g r  r  r  s   r>   r   2TestRsync.test_dir_to_dir_minus_d.<locals>._Check2  r  rA   r  r  rG  rH  r   r  c            
      X   > T R                  [        T R                  SSSTT/SS95        g ra  r  r  s   r>   r  2TestRsync.test_dir_to_dir_minus_d.<locals>._Check3  r  rA   r   r   r   r   c            	      V   > T R                  [        T R                  SSTT/SS95        g r[  r  r  s   r>   r'  2TestRsync.test_dir_to_dir_minus_d.<locals>._Check4  r  rA   r  )r;   r  r  r   r   r  r   r   r  r'  r  r  s   `         @@r>   test_dir_to_dir_minus_d!TestRsync.test_dir_to_dir_minus_d  s      "G  "Gggll7I.Gggll7I.GHHWHHWw&7Kw'HMw"(!0  2 	w'HMw&7Kw"(!0  2 	NNGT7G45w 0 0 9:Hw 0 0 9:HXs#GHI 	Xs#GHI 
bggll7G,	-
w		!++- 89 
.	bggll7G,	-
w		!++- 89 
.Q I 	NNGT4':;w 0 0 9:Hw 0 0 9:HXs#GHI 	Xs#GHI 
bggll7G,	-
w		!++- 89 
.	bggll7G,	-
w		!++- 89 
.
. I 	w&7Kw&7KIIbggll7F+,IIbggll7G,-NNGT4':;w 0 0 9:Hw 0 0 9:HXs#GHI 	Xs#GHI. I   "G  "Gw&7Kw&7Kw&7KNNGT7G45w 0 0 9:Hw 0 0 9:HXsGW#567XsGW#567Q IA 
.	-	-	-4 
.	-	-	-s0   $0V*0V<;0W0W *
V9<
W
W 
W/c                   ^ ^^ T R                  5       mT R                  5       m[        SS5       H0  nT R                  TSU-  SUS-   S9  T R                  TSU-  SUS9  M2     S	S
[        (       a  SOS4/n[	        U5         T R                  SSTT/5        SSS5        [        TT R                  T5      5      n[        TT R                  T5      5      nT R                  X45        [        SS5       H  nT R                  US-   [        [        R                  R                  [        R                  R                  TSU-  5      5      5      5        [        [        R                  R                  TSU-  5      5       nT R                  SSR                  UR                  5       5      5        SSS5        M     U UU4S jnU" 5         g! , (       d  f       GN9= f! , (       d  f       M  = f)zBTests concurrently building listing from multiple tmp file ranges.r   r  zd1-%s   xr   rh  zd2-%s   yrk   rsync_buffer_lines502r
   r  Nxr0   c            	      V   > T R                  [        T R                  SSTT/SS95        g r[  r  r  s   r>   _CheckITestRsync.test_dir_to_dir_minus_d_more_files_than_bufsize.<locals>._CheckD  r  rA   )rM   ranger  r)   r   r   r   rr  r5   r   r   rY   r  ry   r  r  )	r;   irsync_buffer_configr   r   r  r  r  r  s	   `      @@r>   /test_dir_to_dir_minus_d_more_files_than_bufsize9TestRsync.test_dir_to_dir_minus_d_more_files_than_bufsize"  s      "G  "G1d^
$+aK#'!"Q  ) $+aK#' !  #  %&:$.JDC9 : 
1	2
nngtWg67 
3w 0 0 9:Hw 0 0 9:HX(1d^

a%bgg&&Wgk(JLMOWgk23qdii67 43 Q H' 
3	2 43s   G
>0G

G
G+	c                    U R                  SSS9nU R                  5       nU R                  5       nU R                  SSU[	        U5      /5        U R                  S[	        U5      U/SS9n[        [        R                  R                  US5      S	5       nU R                  SUR                  5       5        S S S 5        U R                  S
U5        g ! , (       d  f       N!= f)N   foobar)r   ri  cpz-Zr
   Tr   rbz%bar has a compressed content-encoding)r  rN   rM   r   rQ   r  r   rY   ry   r5   readr{   )r;   	temp_filerZ   r   r   fps         r>   &test_bucket_to_dir_compressed_encoding0TestRsync.test_bucket_to_dir_compressed_encodingL  s    ##Vu#EI""$J!FNND$	4
+;<=^^Wd:&6?*.  0F 
bggll65)4	0B
vrwwy) 
1MM96B 
1	0s   !C
Cc                 H  ^ ^^ T R                  5       mT R                  5       m[        R                  R	                  TS5      n[        R
                  " U5        T R                  TSSSS9  T R                  TSSSS9  T R                  TSS	S
9  T R                  TSSS
9  T R                  TSSSS9  T R                  TSSSS9  T R                  TSSSS9  T R                  TSSSS9  T R                  TSSS
9  [        T R                  TR                  SS5      5      nT R                  TSSSS9  T R                  TSSSS 9  T R                  TSS!SS 9  T R                  US"S#SS 9  T R                  TSSSS 9  T R                  TSS$SS 9  T R                  TSSSS 9  T R                  TSS%SS 9  T R                  TSS&US 9  T R                  TSS'SS 9  [        [        S(S)S*9UU U4S+ j5       nU" 5         U U4S, jnU" 5         [        [        S(S)S*9UU U4S- j5       nU" 5         g.)/z-Tests bucket to dir with mtime at the source.rg  r   r      r   r9  r:  r;  r<  r   r   s   OBJ4r  r  2   rG  rH  obj8   obj8r  obj9   obj9   obj10   obj10r  obj11s   obj11_K   r=  r  rh  r   r   r  s   OBJ7s   OBJ9s   OBJ10   obj11rF   r   r   c                    > TR                  SS[        T5      T/5        [        [        T5      TR                  T5      5      n [        TTR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        [        [        R                  R                  TS5      5       nTR                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  TS5      5       nTR                  S	SR                  UR                  5       5      5        SSS5        [        [        R                  R                  TS
5      5       nTR                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  TS5      5       nTR                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  TS5      5       nTR                  SSR                  UR                  5       5      5        SSS5        g! , (       d  f       GN= f! , (       d  f       GNE= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       g= f)r   r
   r  
r   r?  r@  r   r  rL  /obj8/obj9/obj10/obj11)
r   r?  r   rA  r  rL  r  r  r  r  r9  r0   Nr   OBJ4r  OBJ9r  OBJ10r  obj11_r   rQ   r   r   rr  r5   r   r  r   rY   ry   r  r  s      r>   r   3TestRsync.test_bucket_to_dir_mtime.<locals>._Check1  s    nngtT*%5v>?j)4+>+>z+JKh!1!1&!9:h


   

   VW-.!$))AKKM":; / VV,-1;;=!9: .
 VV,-1;;=!9: . VW-.!$))AKKM":; /VW-.!499Q[[]#;< /. /. .-
 .- /...s<   0I?#0J0J#%0J40K?
J
J #
J14
K
Kc            
      b  > T R                  [        [        R                  R	                  [        R                  R                  TS5      5      5      S5        T R                  [        [        R                  R	                  [        R                  R                  TS5      5      5      S5        T R                  [        [        R                  R	                  [        R                  R                  TS5      5      5      S5        T R                  [        [        R                  R	                  [        R                  R                  TS5      5      5      S5        T R                  [        [        R                  R	                  [        R                  R                  TS5      5      5      S	5        g
)z0Verify mtime was set for objects at destination.r   r  r9  r  r  r  r  r  r  N)r5   r   r   rY   r  ry   )r;   r   s   r>   r   3TestRsync.test_bucket_to_dir_mtime.<locals>._Check2  s   
tBGG,,RWW\\&&-IJKQO
tBGG,,RWW\\&'-JKLaP
tBGG,,RWW\\&&-IJKRP
tBGG,,RWW\\&&-IJK
tBGG,,RWW\\&&-IJKRPrA   c            
      >  > TR                  SSSS[        T5      T/5        [        [        T5      TR                  T5      5      n [        TTR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        TR                  STR                  S[        TS5      /SS	95        TR                  TR                  SS
5        TR                  [        [        R                  R                  [        R                  R                  TS5      5      5      S5        [        [        R                  R                  TS5      5       nTR                  SSR                  UR                  5       5      5        SSS5        [        [        R                  R                  TS5      5       nTR                  SSR                  UR                  5       5      5        SSS5        g! , (       d  f       Np= f! , (       d  f       g= f)r*  r
   r   r  r+  r  rG  r#  Tr$  5r  r  r0   Nr  )r   rQ   r   r   rr  r5   r   r   r   r   r   rY   r  ry   r  r  r  s      r>   r  3TestRsync.test_bucket_to_dir_mtime.<locals>._Check3  s    nngtT4j1A6JKj)4+>+>z+JKh!1!1&!9:h


   

   

..%j&!9:$.
OQ j44fcB
tBGG,,RWW\\&&-IJKQO VV,-1;;=!9: .VW-.!$))AKKM":; /. .-..s   #0G=0H=
H
HN)rN   rM   r   rY   ry   r~  r   r!   r   r   r  r'   r   )r;   rg  time_createdr   r   r  rZ   r   s   `     @@r>   test_bucket_to_dir_mtime"TestRsync.test_bucket_to_dir_mtimeY  s    ""$J!FWW\\&(+FHHV"(&   	")'   	"/-  / 	"(&  ( 	"(&    	"(&   	"(&  ! 	"(&    	")'  ) *"":#9#97#0	23L 	")(    	v")!)   " 	v"(!(!  # 	v"(!/   " 	v"(!(   " 	v"(!(   " 	v"(!(   " 	v"(!(   " 	v")!)*  , 	v")!)   " >3&= 4&=P IQ I >3!< 4!<F IrA   c                 B  ^ ^^^^	^
^^^^^^^^^^^^^ T R                  5       mT R                  5       m[        R                  " T5      R                  n[
        R                  " 5       n[        R                  R                  TS5      n[        R                  " U5        T R                  TSSS[        R                  " 5       S9mT R                  TSS[        5       SS	9mT R                  TS
SS9  T R                  TSS[        5       SS	9mT R                  TSSUSS	9mT R                  TSS[        5       S9mT R                  TSS[        5       SS9mT R                  TSS[        5       [        5       S9mT R                  TSS[        5       [        5       SS 9m	T R                  TS!S"[        5       [        S#9m
T R                  TS$S%[        5       US&S'9mT R                  TS(S)[        [        5       S#9mT R                  TS*S+[        [        5       S,S'9mT R                  TS-S.[        S/S9mT R                  TS0S1[        USS'9mT R                  TS2S3[        USS'9mT R                  TS4S5SS69mT R                  TSS7S89  T R                  TS9S:S89  T R                  US;S<S89  [!        ["        S=S>S?9UUUU	U
UUUUUUUUUUUUU U4S@ j5       nU" 5         T R%                  T R&                  TR(                  S[*        SA5        [!        ["        S=S>S?9UU U4SB j5       nU" 5         gC)DzETests that rsync -P works properly with files that would be orphaned.rg  r   r   r   )rZ   r   r   r   r   r9  r:  540rZ   r   r   r   r   r;  r<  r   r  r  440rG  rH  333r  r  rZ   r   r   r   r  r  777rZ   r   r   r   r   r  r  )rZ   r   r   r   r   r  r  544)rZ   r   r   r   r   r   obj12   obj12rZ   r   r   r   r   obj13   obj13644r   obj14   obj14obj15s   obj15655obj16s   obj16244obj17s   obj17obj18s   obj18obj19s   obj19r   r=  r  r   r   r   r  rF   r   r   c                  
  > TR                  SSS[        T5      T/SSS9n TR                  (       Ga  SnTR                  X5        SnS	nS
nTR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        TR                  XR	                  S5      5        GOTR                  [        U 5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T	[        5      5        TR                  U [        T
[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        TR                  U [        T[        5      5        [        [        T5      TR                  T5      5      n[        TTR                  T5      5      nTR                  U[!        / SQ5      5        TR                  (       a  TR                  U[!        / SQ5      5        gTR                  U[!        / SQ5      5        g)zATests that an exception is thrown because files will be orphaned.r
   ru   r   r   Tr   r   zFor preserving POSIX with rsync downloads, gsutil aborts if a single download will result in invalid destination POSIX. However, thisz9{}#\d+\. User \d+ owns file, but owner does not have readz5{}#\d+ metadata doesn't exist on current system\. GIDz5{}#\d+ metadata doesn't exist on current system\. UIDr   r   r  rG  r  r  r  r  r  r  r  r  r!  r#  r$  r%  r   r?  r@  r  rL  r  r  r  r  /obj12/obj13/obj14z/obj15z/obj16z/obj17z/obj18z/obj19r?  r   r@  rA  r?  r   rA  N)r   rQ   r   r   r|   r{   r   r   r   r   r   r   r   r   rr  r5   r   )r   gcloud_preserve_posix_warning
read_regex	gid_regex	uid_regexr   r   rZ   r   r  r  r  r  r  r  r!  r#  r$  r%  r   r  rG  r  r  r;   r   s          r>   r   CTestRsync.test_bucket_to_dir_preserve_posix_errors.<locals>._Check1  s    ~~D$Z 0&
9  f 
	!	!	!Q 	& 	?Q
L	L	!2!26!:;!1!1&!9:!1!1&!9:!2!26!:;!1!1&!9:!1!1&!9:!1!1'!:;!1!1'!:;!1!1'!:;!1!1'!:;!1!1'!:;!1!1'!:;!2!27!;<!2!27!;<!2!27!;<!2!27!;<mV,7G!HI!GH!GH7G!HI!GH!GH!HI!HI!HI!HI!HI!HIOE+JK	M8H!IJ8H!IJ8H!IJj)4+>+>z+JKh!1!1&!9:h


   
	!	! 	cMN	P 	3'J#KLrA   640c                    > TR                  SSS[        T5      T/SSS9n TR                  (       a  TR                  SU 5        OTR                  [        U 5        [        [        T5      TR                  T5      5      n[        TTR                  T5      5      nTR                  U[        / SQ5      5        TR                  (       a  TR                  U[        / S	Q5      5        gTR                  U[        / S
Q5      5        g)zATests that a file with a valid mode in metadata, nothing changed.r
   ru   r   r   Tr'  z%doesn't exist on current system. GID:r(  r,  r-  N)
r   rQ   r   r{   r   r   r   rr  r5   r   )r   r   r   rZ   r;   r   s      r>   r   CTestRsync.test_bucket_to_dir_preserve_posix_errors.<locals>._Check2  s     ~~D$Z 0&
9  f 
	!	!=vFmV,j)4+>+>z+JKh!1!1&!9:h


   
	!	!cMN	P 	3'J#KLrA   N)rN   rM   r   statst_gidr+   r   rY   ry   r~  r   getuidr-   r.   r/   r  r'   r   r  r   r   r#   )r;   r   r   rg  r   r   rZ   r   r  r  r  r  r  r  r!  r#  r$  r%  r   r  rG  r  r  r   s   `     @@@@@@@@@@@@@@@@@@r>   (test_bucket_to_dir_preserve_posix_errors2TestRsync.test_bucket_to_dir_preserve_posix_errors  s    ""$J!F''&/((K++-OWW\\&(+FHHV
)/&-"'!#	  .D 
)0&.!,"'	  )D
 	"/-  / 
)/&-!,"'	  )D 
)/&-!0"'	  )D 
)/&-!,  0D
 
)/&-!,"'	  )D *1'/"--"--	  1E *1'/"--"--#(  *E *1'/"--")	  +E *1'/"--"-#(  *E *1'/")"--	  1E *1'/")"--#(  *E *1'/")#(	  *E *1'/")"-#(  *E *1'/")"1#(  *E *1'/#(  *E 	v8Lv'Jv"(!/  1
 >3DM DM DM 4DML I 	**4+@+@+5+A+A7+4e=
 >3M 4M8 IrA   c           	        ^ ^^ T R                  5       mT R                  5       m[        R                  " T5      R                  n[
        R                  " 5       n[        R                  R                  TS5      n[        R                  " U5        T R                  TSSSS9  T R                  TSSUS9  T R                  TS	S
US9  T R                  TSSUSS9  T R                  TSSUSS9  T R                  TSS[        S9  T R                  TSS[        SS9  T R                  TSS[        US9  T R                  TSS[        US9  T R                  TSS[        USS 9  T R                  TS!S"[        US#S 9  T R                  TS$S%[        SS9  T R                  TSS&S'9  T R                  TS(S)S'9  T R                  US*S+S'9  [        [        S,S-S.9UU U4S/ j5       nU" 5         T R                  [        R                  R                  TS5      [        R                   " 5       S0S19  T R                  [        R                  R                  TS5      U[        R                   " 5       ["        S29  T R                  [        R                  R                  US35      U["        S49  T R                  [        R                  R                  TS5      US5S49  T R                  [        R                  R                  TS5      US0S49  T R                  [        R                  R                  TS5      U["        S49  T R                  [        R                  R                  TS5      [        S6S19  T R                  [        R                  R                  TS5      [        U["        S79  T R                  [        R                  R                  TS5      [        U["        S79  T R                  [        R                  R                  TS5      [        US8S79  T R                  [        R                  R                  TS!5      [        US9S79  T R                  [        R                  R                  TS$5      [        S0S19  g:);z@Tests that rsync -P works properly with default file attributes.rg  r   r   r   r   r9  r:  r   r;  r<  r  r  555r  rG  rH  r  r  r  r  r  422r  r  r  r  r  r  r  r  400r   r  r  533r  r  r=  r  r   r   r   r  rF   r   r   c                  6  > TR                  SSS[        T5      T/5        [        [        T5      TR                  T5      5      n [        TTR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        g)zCVerifies that all attributes were copied correctly when -P is used.r
   ru   r   )r   r?  r@  r  rL  r  r  r  r  r)  r*  r+  )r   r?  r@  r   rA  r  rL  r  r  r  r  r)  r*  r+  Nr   rQ   r   r   rr  r5   r   r|  s     r>   r   FTestRsync.test_bucket_to_dir_preserve_posix_no_errors.<locals>._Check1?  s     nngtT4
+;VDEj)4+>+>z+JKh!1!1&!9:h


   

  rA   i$  )r   r   )r   r   r   r   )r   r   im  i  )r   r   r      i[  N)rN   rM   r   r6  r7  r+   r   rY   ry   r~  r   r/   r  r'   r   VerifyLocalPOSIXPermissionsr8  r,   )r;   r   r   rg  r   rZ   r   s   `    @@r>   +test_bucket_to_dir_preserve_posix_no_errors5TestRsync.test_bucket_to_dir_preserve_posix_no_errors  s   
 ""$J!F''&/((K++-OWW\\&(+FHHV"(&   " 	")'%  ' 	"/-)  + 	"(&% 	  "
 	"(&) 	  "
 	"(&!  # 	"(&! 	  "
 	")'!%	  '
 	")'!)	  +
 	")'!%   " 	")'!)   " 	")'! 	  "
 	v8Lv'Jv"(!/  1
 >3 4* I$$RWW\\&&%A)+*/ % 1 	$$RWW\\&'%B)4)+*6 % 8 	$$RWW\\&&%A)8*6 % 8 	$$RWW\\&&%A)4*/ % 1 	$$RWW\\&&%A)8*/ % 1 	$$RWW\\&&%A)4*6 % 8 	$$RWW\\&&%A)0*/ % 1 	$$RWW\\&'%B)0)4*6 % 8 	$$RWW\\&'%B)0)8*6 % 8 	$$RWW\\&'%B)0)4*/ % 1 	$$RWW\\&'%B)0)8*/ % 1 	$$RWW\\&'%B)0*/ % 1rA   c                   ^ ^^	 T R                  5       mT R                  5       m	[        R                  R	                  T	S5      n[        R
                  " U5        T R                  TSSS9  T R                  TSSSS9  T R                  TS	S
S9  T R                  T	SSS9  T R                  T	SSS9  T R                  USSS9  [        [        SSS9UU U	4S j5       nU" 5         [        [        SSS9UU U	4S j5       nU" 5         [        [        SSS9UU U	4S j5       nU" 5         [        [        SSS9UU U	4S j5       nU" 5         T R                  TSSS9  T R                  T	SSS9  T R                  S[        TS5      /5        [        R                  " [        R                  R	                  T	S5      5        [        [        SSS9UU U	4S j5       nU" 5         [        [        SSS9UU U	4S j5       nU" 5         g) zBTests that flat and recursive rsync bucket to dir works correctly.rg  r   r   r   r9  r:  r   r   r;  r<  r=  r  r   r   r   r  rF   r   r   c            
      p  > TR                  SS[        T5      T/5        [        [        T5      TR                  T5      5      n [        TTR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        TR                  STR                  S[        TS5      /SS95        [        [        R                  R                  TS5      5       nTR                  SS	R                  UR                  5       5      5        S
S
S
5        g
! , (       d  f       g
= f)r   r
   r  rW  rX  r9  r#  Tr$  r0   Nr  r  s      r>   r   5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check1  s     nngtT*%5v>?j)4+>+>z+JKh!1!1&!9:h
x%H!IJ x%H!IJ 

..%j'!:;'+  -. VW-.!$))AKKM":; /..s   .0D''
D5c                  b   > TR                  SS[        T5      T/SS9n TR                  U 5        g r[  r  rw  s    r>   r   5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check2  s:     ~~D$z*F
34  If
F#rA   c            
      r  > TR                  SSS[        T5      T/5        [        [        T5      TR                  T5      5      n [        TTR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        TR                  STR                  S[        TS5      /SS	95        [        [        R                  R                  TS5      5       nTR                  SS
R                  UR                  5       5      5        SSS5        g! , (       d  f       g= f)r*  r
   r  r+  rW  rX  r9  r#  Tr$  r0   Nr  r  s      r>   r  5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check3  s     nngtT4
+;VDEj)4+>+>z+JKh!1!1&!9:h
x%H!IJ x%H!IJ 

..%j'!:;'+  -. VW-.!$))AKKM":; /..s   /0D((
D6c                  d   > TR                  SSS[        T5      T/SS9n TR                  U 5        g r  r  rw  s    r>   r'  5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check4  <     ~~D$Z 0&
9  Of
F#rA   r  r  rG  rH  rI  c                  6  > TR                  SSS[        T5      T/5        [        [        T5      TR                  T5      5      n [        TTR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        g r^  rA  r|  s     r>   r.  5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check5  s    
nngtT4
+;VDEj)4+>+>z+JKh!1!1&!9:h
x%H!IJ x%H!IJrA   c                  d   > TR                  SSS[        T5      T/SS9n TR                  U 5        g ra  r  rw  s    r>   r  5TestRsync.test_bucket_to_dir_minus_d.<locals>._Check6  rP  rA   N)rN   rM   r   rY   ry   r~  r   r  r'   r   r   rQ   r  )
r;   rg  r   r   r  r'  r.  r  rZ   r   s
   `       @@r>   test_bucket_to_dir_minus_d$TestRsync.test_bucket_to_dir_minus_d  s!    ""$J!FWW\\&(+FHHV"(&  (
 	")'   	"/-  / 	v8Lv'Jv"(!/  1
 >3< 4<( I >3$ 4$ I >3< 4<& I >3$ 4$ I 	"(&  ( 	v'JNND$z6234IIbggll67+, >3K 4K I >3$ 4$ IrA   c                    ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  [	        [
        SSS9UU U4S	 j5       nU" 5         g
)a  Tests that name case changes work correctly.

Example:

Windows filenames are case-preserving in what you wrote, but case-
insensitive when compared. If you synchronize from FS to cloud and then
change case-naming in local files, you could end up with this situation:

Cloud copy is called .../TiVo/...
FS copy is called      .../Tivo/...

Then, if you rsync from cloud to FS, if rsync doesn't recognize that on
Windows these names are identical, each rsync run will cause both a copy
and a delete to be executed.
r   r   r   Obj1r  rF   r   r   c                     > TR                  SSS[        T5      T/SS9n [        (       a  TR                  [        U 5        gTR                  [        U 5        g)r   r
   r  r   Tr   N)r   rQ   r)   r5   r   assertNotEqual)outputrZ   r;   r   s    r>   r   LTestRsync.test_bucket_to_dir_minus_d_with_fname_case_change.<locals>._Check1	  sU     ~~D$Z 0&
9  Of 
V,J/rA   N)rN   rM   r   r  r'   r   )r;   r   rZ   r   s   ` @@r>   1test_bucket_to_dir_minus_d_with_fname_case_change;TestRsync.test_bucket_to_dir_minus_d_with_fname_case_change	  sw    & ""$J!F"(&  ( 	v'J >30 40 IrA   z^The boto lib used for S3 does not handle objects starting with slashes if we use V4 signature.c                    ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TS5      nT R	                  US5        [        [        SSS9UU U4S	 j5       nU" 5         g
)zTests that we correctly handle leftover dir placeholders.

See comments in gslib.commands.rsync._FieldedListingIterator for details.
r   r   r   / rF   r   r   c                  2  > TR                  SSS[        T5      T/SS9  [        [        T5      TR                  T5      5      n [        TTR	                  T5      5      nTR                  U [        SS/5      5        TR                  U[        S/5      5        g)	r   r
   r  r   Tr   r   z//NrA  r|  s     r>   r   STestRsync.test_bucket_to_dir_minus_d_with_leftover_dir_placeholder.<locals>._Check1=	  s     nnD$Z 0&
9  Oj)4+>+>z+JKh!1!1&!9:h
xgt_!56
xgY0rA   N)rN   rM   r   StorageUriCloneReplaceNameStorageUriSetContentsFromStringr'   r   )r;   key_urir   rZ   r   s   `  @@r>   8test_bucket_to_dir_minus_d_with_leftover_dir_placeholderBTestRsync.test_bucket_to_dir_minus_d_with_leftover_dir_placeholder+	  s     ""$J!F"(&  ( --j#>G(("5 >3	1 4	1 IrA   z)os.symlink() is not available on Windows.c           	      d   U R                  5       nU R                  5       n[        R                  R	                  US5      n[        R
                  " U5        [        R
                  " [        R                  R	                  US5      5        [        R                  " [        R                  R	                  US5      [        R                  R	                  US5      5        [        R                  " [        R                  R	                  US5      5        U R                  SSSU[        U5      /5        g)z7Tests that rsync -e -r ignores symlinks when recursing.rg  missingr
   r   -eN)
rN   rM   r   rY   ry   r~  symlinkrmdirr   rQ   )r;   rZ   r   rg  s       r>   test_rsync_minus_r_minus_e$TestRsync.test_rsync_minus_r_minus_eK	  s     ""$J!FWW\\&(+FHHVHHRWW\\&),- JJrww||FI.VY0OPHHRWW\\&),-NNGT4j1ABCrA   c                   ^ ^^	 T R                  5       m	[        R                  R                  T	S5      n[        R                  " U5        T R                  5       mT R                  T	SSS9nT R                  T	SSS9  T R                  USSS9  [        R                  R                  T	S	5      n[        R                  " X#5        [        R                  R                  T	S
5      n[        R                  " [        R                  R                  SS5      U5        T R                  TSSS9  T R                  TSSS9  T R                  TSSS9  [        [        SSS9UU U	4S j5       nU" 5         [        R                  " U5        [        [        SSS9UU U	4S j5       nU" 5         [        [        SSS9UU U	4S j5       nU" 5         g)z%Tests that rsync -e ignores symlinks.rg  r   r   r  r9  r:  r   r<  symlink1symlink2r`  znon-existentr=  r   r   r   r  r  rF   r   r   c            	      6  > TR                  SSST[        T5      /5        [        TTR                  T5      5      n [        [        T5      TR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        g)z1Ensure listings match the commented expectations.r
   r  rk  )r   r?  r@  	/symlink1z	/symlink2rX  Nr  r|  s     r>   r   5TestRsync.test_rsync_minus_d_minus_e.<locals>._Check1x	  s     nngtT64
3CDE!1!1&!9:hj)4+>+>z+JKh


K
LN
 x%H!IJrA   c            
        > TR                  SST[        T5      /5        [        TTR                  T5      5      n [        [        T5      TR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        TR                  STR                  S[        TS5      /SS	95        g
)r   r
   r  )r   r?  r@  rt  )r   r?  rA  rt  r   r#  rq  Tr$  Nr  r|  s     r>   r   5TestRsync.test_rsync_minus_d_minus_e.<locals>._Check2	  s     nngtVT*-=>?!1!1&!9:hj)4+>+>z+JKh
xKLN xKLN


..%j*!=>'+  -.rA   c                  b   > TR                  SST[        T5      /SS9n TR                  U 5        g r[  r  rw  s    r>   r  5TestRsync.test_rsync_minus_d_minus_e.<locals>._Check3	  r  rA   N)rM   r   rY   ry   r~  rN   r  rl  r   r'   r   r  )
r;   rg  fpath1good_symlink_pathbad_symlink_pathr   r   r  rZ   r   s
   `       @@r>   test_rsync_minus_d_minus_e$TestRsync.test_rsync_minus_d_minus_eY	  s    !FWW\\&(+FHHV""$J  +1*1 ! 3F 	v8Lv"(!/  1 VZ8JJv) ww||FJ7JJrww||C02BC")'  ) 	"(&  ( 	"/-  /
 >3K 4K I
 II
>3. 4." I >3$ 4$ IrA   z%S3 does not support composite objectsc           	        ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  S[        TS5      [        TS5      [        TS5      /5        T R                  TSSS9  T R                  TS	S
S9  [	        [
        SSS9UUU 4S j5       nU" 5         [	        [
        SSS9UUU 4S j5       nU" 5         g)zFTests that rsync works with composite objects (which don't have MD5s).r   r   r   r9  r:  composer   r=  r   r   rF   r   r   c                  X  > TR                  SS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        / SQ5      5        TR	                  U[        / SQ5      5        g )Nr
   r  r   r?  r   r   rC  s     r>   r   HTestRsync.test_bucket_to_bucket_minus_d_with_composites.<locals>._Check1	  s    
nngtT+%6[8IJKk*D,?,?,LMhk*D,?,?,LMh
x%A!BC x%A!BCrA   c                  t   > TR                  SS[        T5      [        T5      /SS9n TR                  U 5        g r[  r  r   s    r>   r   HTestRsync.test_bucket_to_bucket_minus_d_with_composites.<locals>._Check2	  rP  rA   NrN   r   r   rQ   r'   r   r;   r   r   r   r   s   `  @@r>   -test_bucket_to_bucket_minus_d_with_composites7TestRsync.test_bucket_to_bucket_minus_d_with_composites	  s    ##%K##%K"(&  ( 	")'  ) 	NN[&!['"[&!	  	")'  ) 	"(&  (
 >3D 4D I >3$ 4$ IrA   c                   ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  [        [        SSS9UUU 4S	 j5       nU" 5         [        [        SSS9UUU 4S
 j5       nU" 5         g)zETests working with empty dest bucket (iter runs out before src iter).r   r   r   r9  r:  rF   r   r   c                  X  > TR                  SS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        SS/5      5        TR	                  U[        SS/5      5        g )Nr
   r  r   r?  r   rC  s     r>   r   CTestRsync.test_bucket_to_bucket_minus_d_empty_dest.<locals>._Check1	  s    
nngtT+%6[8IJKk*D,?,?,LMhk*D,?,?,LMh
xgx%8!9:
xgx%8!9:rA   c                  t   > TR                  SS[        T5      [        T5      /SS9n TR                  U 5        g r[  r  r   s    r>   r   CTestRsync.test_bucket_to_bucket_minus_d_empty_dest.<locals>._Check2	  rP  rA   NrN   r   r'   r   r  s   `  @@r>   (test_bucket_to_bucket_minus_d_empty_dest2TestRsync.test_bucket_to_bucket_minus_d_empty_dest	  s    ##%K##%K"(&  ( 	")'  )
 >3; 4; I >3$ 4$ IrA   c                   ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  [        [        SSS9UUU 4S	 j5       nU" 5         [        [        SSS9UUU 4S
 j5       nU" 5         g)zDTests working with empty src bucket (iter runs out before dst iter).r   r   r   r9  r:  rF   r   r   c                    > TR                  SS[        T5      [        T5      /5        TR                  S[        TS5      /SSS9n TR                  SU 5        TR                  S[        TS5      /SSS9n TR                  SU 5        g )	Nr
   r  ls**r   Tr'  #One or more URLs matched no objects)r   rQ   r{   r   s    r>   r   BTestRsync.test_bucket_to_bucket_minus_d_empty_src.<locals>._Check1
  s    
nngtT+%6[8IJK~~tT+t%<=./,0  2f mm96B~~tT+t%<=./,0  2f mm96BrA   c                  t   > TR                  SS[        T5      [        T5      /SS9n TR                  U 5        g r[  r  r   s    r>   r   BTestRsync.test_bucket_to_bucket_minus_d_empty_src.<locals>._Check2
  rP  rA   Nr  r  s   `  @@r>   'test_bucket_to_bucket_minus_d_empty_src1TestRsync.test_bucket_to_bucket_minus_d_empty_src	  s    ##%K##%K"(&  ( 	")'  )
 >3	C 4	C I >3$ 4$ IrA   c           	      .  ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  SSS[        TS5      /5        [	        [
        SSS	9UUU 4S
 j5       nU" 5         [	        [
        SSS	9UUU 4S j5       nU" 5         g)z#Tests that rsync -p preserves ACLs.r   r   r   aclr   rt   rF   r   r   c            	        > TR                  SSS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        S/5      5        TR	                  U[        S/5      5        TR                  SS[        TS5      /SS	9nTR                  SS[        TS5      /SS	9nTR	                  X#5        g
)z!Tests rsync -p works as expected.r
   r  -pr   r  getr   Tr$  Nr   r   r   	acl1_json	acl2_jsonr   r   r;   s       r>   r   5TestRsync.test_rsync_minus_d_minus_p.<locals>._Check10
  s     nnD$ k*D,?,?,LMhk*D,?,?,LMh
xgY0
xgY0..%k62
34 ! Ii..%k62
34 ! Ii
y,rA   c            	      v   > TR                  SSS[        T5      [        T5      /SS9n TR                  U 5        g )Nr
   r  r  Tr   r  r   s    r>   r   5TestRsync.test_rsync_minus_d_minus_p.<locals>._Check2D
  rc  rA   Nr  r  s   `  @@r>   test_rsync_minus_d_minus_p$TestRsync.test_rsync_minus_d_minus_p%
  s    ##%K##%K"(&  ( 	NNE5-k61JKL >3- 4-  I >3$ 4$ IrA   c                    ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  SS[        TS5      /5        [	        [
        SSS9UUU 4S	 j5       nU" 5         g
)z!Tests that rsync -a applies ACLs.r   r   r   r  r  rF   r   r   c            
      ,  > TR                  SSSS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        S/5      5        TR	                  U[        S/5      5        TR                  SSS[        TS5      /5        TR                  SS	[        TS5      /S
S9nTR                  SS	[        TS5      /S
S9nTR	                  X#5        g)z!Tests rsync -a works as expected.r
   r  rs   rt   r   r  r   r   r  Tr$  Nr   r  s       r>   r  /TestRsync.test_rsync_canned_acl.<locals>._CheckZ
  s    nn
4}
{

{
 	
 k*D,?,?,LMhk*D,?,?,LMh
xgY0
xgY0 nneUM4V3LMN..%k62
34 ! Ii..%k62
34 ! Ii
y,rA   Nr  )r;   r  r   r   s   ` @@r>   test_rsync_canned_aclTestRsync.test_rsync_canned_aclP
  sz    ##%K##%K"(&  ( 	NNE5${F";<= >3- 4-( HrA   c                   ^ ^^ T R                  5       m[        R                  R                  TS5      n[        R                  " U5        T R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  USSS9  [        [        S	S
S9UU U4S j5       nU" 5         [        [        S	S
S9UU U4S j5       nU" 5         g)z5Tests that rsync to non-existent bucket subdir works.rg  r   r   r  r9  r:  r   r<  rF   r   r   c            	      X  > TR                  SST[        TS5      /5        [        TTR                  T5      5      n [        [        TS5      TR	                  TR                  TS5      5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        g)r   r
   r   rg  rW  Nr   rQ   r   rr  r   rd  r5   r   r   r   
bucket_urlr;   r   s     r>   r   BTestRsync.test_rsync_to_nonexistent_bucket_subdir.<locals>._Check1
  s     nngtVT*h-GHI!1!1&!9:h
z8
$


--j(CEFh
 x%H!IJ
x%H!IJrA   c            	      d   > TR                  SST[        TS5      /SS9n TR                  U 5        g )Nr
   r   rg  Tr   r  r   r  r;   r   s    r>   r   BTestRsync.test_rsync_to_nonexistent_bucket_subdir.<locals>._Check2
  sA     ~~D&
H%'  f F#rA   N)	rM   r   rY   ry   r~  rN   r  r'   r   )r;   rg  r   r   r  r   s   `   @@r>   'test_rsync_to_nonexistent_bucket_subdir1TestRsync.test_rsync_to_nonexistent_bucket_subdirq
  s     !FWW\\&(+FHHV""$Jv'Jv8Lv"(!/  1
 >3K 4K I >3$ 4$ IrA   c                 4  ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  TSSS9  [	        [
        SS	S
9UU U4S j5       nU" 5         [	        [
        SS	S
9UU U4S j5       nU" 5         g)aH  Tests that rsync with destination url as a prefix of existing obj works.

Test to make sure that a dir/subdir gets created if it does not exist
even when the new dir is a prefix of an existing dir.
e.g if gs://some_bucket/foobar exists, and we run the command
rsync some_dir gs://some_bucket/foo
this should create a subdir foo
foobarr   r   r   r  r9  r:  rF   r   r   c            	      X  > TR                  SST[        TS5      /5        [        TTR                  T5      5      n [        [        TS5      TR	                  TR                  TS5      5      5      nTR                  U [        SS/5      5        TR                  U[        SS/5      5        g)r   r
   r   foor   r?  Nr  r  s     r>   r   YTestRsync.test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_obj.<locals>._Check1
  s     nngtVT*e-DEF!1!1&!9:h
z5
!


d==%! "#h
 xgx%8!9:
xgx%8!9:rA   c            	      d   > TR                  SST[        TS5      /SS9n TR                  U 5        g )Nr
   r   r  Tr   r  r  s    r>   r   YTestRsync.test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_obj.<locals>._Check2
  s@     ~~wf#J68,0  2f F#rA   NrM   rN   r   r  r'   r   )r;   r   r   r  r   s   `  @@r>   >test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_objHTestRsync.test_rsync_to_nonexistent_bucket_subdir_prefix_of_existing_obj
  s     !F""$J"*&  ( 	v'Jv8L >3; 4; I >3$ 4$ IrA   c                    ^ ^^ T R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  < ST R                  < 3m[	        [
        SSS	9UU U4S
 j5       nU" 5         g)DTests that rsync from a non-existent bucket subdir fails gracefully.r   r   r  r9  r:  ://rF   r   r   c                    > TR                  SSTT/SSS9n TR                  (       a  TR                  SU 5        OTR                  SU 5        [        TTR	                  T5      5      nTR                  U[        SS	/5      5        g 
Nr
   r  r   Tr'  znot found: 404zCaught non-retryable exceptionr   r?  r   r   r{   r   rr  r5   r   r   listingbucket_url_strr;   r   s     r>   r  <TestRsync.test_rsync_from_nonexistent_bucket.<locals>._Check
      ~~wnfE./,0  2f 
	!	!&/6? 0 0 89g
wWh$7 89rA   NrM   r  r   r   r'   r   r;   r  r  r   s   ` @@r>   "test_rsync_from_nonexistent_bucket,TestRsync.test_rsync_from_nonexistent_bucket
  y    !Fv'Jv8L"&"7"7"&">">@N >3
: 4
: HrA   c                    ^ ^^ T R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  < ST R                  < 3m[	        [
        SSS	9UU U4S
 j5       nU" 5         g)r  r   r   r  r9  r:  r  rF   r   r   c                    > TR                  SSTT/SSS9n TR                  (       a  TR                  SU 5        OTR                  SU 5        [        TTR	                  T5      5      nTR                  U[        SS	/5      5        g r  r  r  s     r>   r  :TestRsync.test_rsync_to_nonexistent_bucket.<locals>._Check
  r  rA   Nr  r  s   ` @@r>    test_rsync_to_nonexistent_bucket*TestRsync.test_rsync_to_nonexistent_bucket
  r  rA   c                   ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  TSSS9  T R                  TSSS9  T R                  TS5        T R                  TS5        [        [        SS	S
9UUU 4S j5       nU" 5         [        [        SS	S
9UUU 4S j5       nU" 5         g)z<Tests that punc chars in filenames don't confuse sort order.e/obj1r   r   	e-1/.obj2r:  r  rE   rF   r   r   c            
        > TR                  SS[        T5      [        T5      /5        [        [        T5      TR                  T5      5      n [        [        T5      TR                  T5      5      nTR	                  U [        SS/5      5        TR	                  U[        SS/5      5        TR	                  STR                  S[        TS5      /SS	95        TR	                  S
TR                  S[        TS5      /SS	95        g)r   r
   z-rdz/e/obj1z
/e-1/.obj2r   r#  r  Tr$  r9  r  Nr   rC  s     r>   r   VTestRsync.test_bucket_to_bucket_minus_d_with_overwrite_and_punc_chars.<locals>._Check1  s     nngud;&7k9JKLk*D,?,?,LMhk*D,?,?,LMh
xi%>!?@
xi%>!?@


..%k8!<='+  -. 

..%k;!?@'+  -.rA   c                  t   > TR                  SS[        T5      [        T5      /SS9n TR                  U 5        g r[  r  r   s    r>   r   VTestRsync.test_bucket_to_bucket_minus_d_with_overwrite_and_punc_chars.<locals>._Check2&  rP  rA   N)rN   r   r  r'   r   r  s   `  @@r>   ;test_bucket_to_bucket_minus_d_with_overwrite_and_punc_charsETestRsync.test_bucket_to_bucket_minus_d_with_overwrite_and_punc_chars
  s    ##%K##%K 	"*&  ( 	"-'  ) 	"*&  ( 	"-'  )
 	Q/Q/ >3. 4.$ I >3$ 4$ IrA   c                   ^ ^^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  TSSS9  T R                  TSSS9  T R                  TS	S
S9  T R                  TSSS9  T R	                  TS5        [        [        SSS9UUU U4S j5       nU" 5         [        [        SSS9UUU U4S j5       nU" 5         g)z1Tests that rsync regex exclusions work correctly.r   r   r  r9  r:  r   r   r   r   r   r   r   rF   r   r   c            
      8  > TR                  SSTST[        T5      /5        [        TTR                  T5      5      n [        [        T5      TR	                  T5      5      nTR                  U [        / SQ5      5        TR                  U[        / SQ5      5        g)r   r
   r  obj[34]r  )r   r?  r   Nr  )r   r   rZ   flagr;   r   s     r>   r   BTestRsync._test_dir_to_bucket_regex_paramaterized.<locals>._Check1H  s     nngtT9fd:>NOP!1!1&!9:hj)4+>+>z+JKh
x%A!BC x%A!BCrA   c            
      f   > TR                  SSTST[        T5      /SS9n TR                  U 5        g )Nr
   r  r  Tr   r  )r   rZ   r  r;   r   s    r>   r   BTestRsync._test_dir_to_bucket_regex_paramaterized.<locals>._Check2W  sC     ~~D$	6
  f F#rA   N)rM   rN   r  r   r  r'   r   )r;   r  r   r   rZ   r   s   ``  @@r>   '_test_dir_to_bucket_regex_paramaterized1TestRsync._test_dir_to_bucket_regex_paramaterized1  s    !F""$Jv'Jv8Lv'J")'  ) 	"(&  ( 	"(&  ( 	
A. >3	D 4	D I >3$ 4$ IrA   c                 &    U R                  S5        g)z8Tests that rsync regex exclusions work correctly for -x.-xNr  r;   s    r>   test_dir_to_bucket_minus_x$TestRsync.test_dir_to_bucket_minus_xb      006rA   c                 &    U R                  S5        g)z8Tests that rsync regex exclusions work correctly for -y.-yNr  r  s    r>   test_dir_to_bucket_minus_y$TestRsync.test_dir_to_bucket_minus_yf  r  rA   c           
         U R                  5       nU R                  / SQS9nU R                  SSUSU[        U5      /SS9  [	        X@R                  U5      5      nU R                  U[        / SQ5      5        U(       aB  [	        [        U5      U R                  U5      5      nU R                  U[        / S	Q5      5        gU R                  S
[        US5      /SSS9nU R                  SU5        g)z3Tests if negative lookahead includes files/objects.abc)data1za.txt)r  ok)data2zb.txt)data3data4zc.txtr  r
   r   z^(?!.*\.txt$).*Tr   /a/b/c/data1/a.txt	/data1/ok/data2/b.txt/data3/data4/c.txt)r  r  r  r  r  r   r'  r  N)
rN   rM   r   rQ   r   rr  r5   r   r   r{   )r;   r  includesrZ   r   r  actualr   s           r>   ,_test_dir_to_bucket_regex_negative_lookahead6TestRsync._test_dir_to_bucket_regex_negative_lookaheadj  s   ""$J , F 	NN	$0&	j		   f..v67G 
 	 tJ')<)<Z)HIf

#LMO ~~tT*d%;<./,0  2f mm96BrA   c                 $    U R                  SSS9  g)z=Test that rsync -x negative lookahead includes objects/files.r  Tr  Nr	  r  s    r>   6test_dir_to_bucket_negative_lookahead_works_in_minus_x@TestRsync.test_dir_to_bucket_negative_lookahead_works_in_minus_x  s    55dT5JrA   c                 $    U R                  SSS9  g)zFTest that rsync -y nevative lookahead does not includes objects/files.r  Fr  Nr  r  s    r>   7test_dir_to_bucket_negative_lookahead_breaks_in_minux_yATestRsync.test_dir_to_bucket_negative_lookahead_breaks_in_minux_y  s    55dU5KrA   c                    ^ ^^^ T R                  / SQS9m[        [        SSS9UU U4S j5       mU4S jnU" 5         UU 4S jnU" 5         UU U4S	 jnU(       a  U" 5         g
g
)zFTest that rsync regex options work with a relative regex per the docs.r  r  rF   r   r   c                 l  > TR                  5       nSnT	[        (       a  SOS-   nUTR                  SSTX[        U5      /SS9-  n[	        T	TR                  T	5      5      nTR                  U[        / SQ5      5        [	        [        U5      TR                  U5      5      nTR                  Xa5        U$ )	z'Tests rsync skips the excluded pattern.ra  \r`  r
   r   Tr   r  )	rN   r)   r   rQ   r   rr  r5   r   r   )
exclude_regexexpectedrZ   r   localr  r  r  r;   r   s
          r>   _check_exclude_regexXTestRsync._test_dir_to_bucket_relative_regex_paramaterized.<locals>._check_exclude_regex  s     $$&jf

4eD$
   f
  0 0 89g


   tJ')<)<Z)HIf
v(mrA   c                  .   > T " S[        / SQ5      5        g)zCEnsure the example exclude pattern from the docs works as expected.zdata.[/\\].*\.txt$)r   r  r  r  N)r   )r  s   r>   r   KTestRsync._test_dir_to_bucket_relative_regex_paramaterized.<locals>._Check1  s    2>?ArA   c                     > TR                   (       a  [        (       a  T " S[        S/5      5        gT " S[        S/5      5        g)z1Tests that a regex with a pipe works as expected.z^data^^^|[bc]$r   z^data|[bc]$N)r   r)   r   )r  r;   s   r>   r   KTestRsync._test_dir_to_bucket_relative_regex_paramaterized.<locals>._Check2  s3    		!	!jj 	-sD6{;]CK8rA   c            	      ,  > T" S[        / SQ5      5      n TR                  SR                  [        R                  R                  TS5      5      U 5        TR                  SR                  [        R                  R                  TSS5      5      U 5        g)z>Tests that directories are skipped from iteration as expected.r  )r   r  r  r  r  r  z!Skipping excluded directory {}...r  N)r   r{   r|   r   rY   ry   r   )r   r  r;   r   s    r>   r  KTestRsync._test_dir_to_bucket_relative_regex_paramaterized.<locals>._Check3  s    #

M
NPf mm
-
4
4ggll67+-.46 
-
4
4ggll67G467=?rA   N)rM   r'   r   )r;   r  	skip_dirsr   r   r  r  r   s   ``    @@r>   0_test_dir_to_bucket_relative_regex_paramaterized:TestRsync._test_dir_to_bucket_relative_regex_paramaterized  se     , F >3 4,A
 I9 I
? i rA   c                 $    U R                  SSS9  g)zCTest that rsync -x option works with a relative regex per the docs.r  Fr!  Nr"  r  s    r>   #test_dir_to_bucket_relative_minus_x-TestRsync.test_dir_to_bucket_relative_minus_x  s    99$%9PrA   c                 $    U R                  SSS9  g)zCTest that rsync -y option works with a relative regex per the docs.r  Tr%  Nr&  r  s    r>   #test_dir_to_bucket_relative_minus_y-TestRsync.test_dir_to_bucket_relative_minus_y  s    99$$9OrA   z1os.chmod() won't make file unreadable on Windows.c                 $  ^ ^^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9m[        R                  " TS5        T R                  TSSS9  [        [        S	S
S9UUU U4S j5       nU" 5         g)z+Tests that rsync -C option works correctly.r   r   r  r   r   r   r   r   rF   r   r   c                    > TR                  SST[        T5      /SSS9n TR                  (       a"  TR                  SR	                  T5      U 5        OTR                  SU 5        [        TTR                  T5      5      n[        [        T5      TR                  T5      5      nTR                  U[        / SQ5      5        TR                  U[        S	S
/5      5        g)r   r
   z-Cr   Tr'  zPermission denied: '{}'z,1 files/objects could not be copied/removed.)r   r   r   r   r   N)
r   rQ   r   r{   r|   r   rr  r   r5   r   )r   r   r   rZ   rY   r;   r   s      r>   r  4TestRsync.test_dir_to_bucket_minus_C.<locals>._Check  s     ~~D&$z"2
3  f 
	!	!/66t<fEDfM!1!1&!9:hj)4+>+>z+JKh
x%@!AB
xgw%7!89rA   N)rM   rN   r  r   chmodr'   r   )r;   r  rZ   rY   r   s   ` @@@r>   test_dir_to_bucket_minus_C$TestRsync.test_dir_to_bucket_minus_C  s    
 !F""$Jv'Jf)/(/  1D HHT1v'J >3: 4:" HrA   z5Windows Unicode support is problematic in Python 2.x.c                   ^ ^^^ T R                  5       mT R                  5       m/ SQnU H  nT R                  TUS9  M     [        (       a  [	        SS/5      O[	        / SQ5      m[        [        SSS9UUU U4S	 j5       nU" 5         g
)z;Tests that rsync -r works correctly with unicode filenames.)u   morales_suenÌƒos.jpgu   morales_suenÌƒos.jpgu   fooꝾoo)r   ri     /morales_suenÌƒos.jpg	   /fooꝾoo)u   /morales_suenÌƒos.jpgr3  r4  rF   r   r   c                  ,  > TR                  SST[        T5      /5        [        TTR                  T5      5      n [        [        T5      TR	                  T5      5      nTR                  [        U 5      T5        TR                  [        U5      T5        g)r   r
   r   Nr  )r   r   rZ   expected_list_resultsr;   r   s     r>   r  ?TestRsync.test_dir_to_bucket_with_unicode_chars.<locals>._Check  s|     nngtVT*-=>?!1!1&!9:hj)4+>+>z+JKh
s8}&;<
s8}&;<rA   N)rM   rN   r  r(   	frozensetr'   r   )r;   	file_listfilenamer  rZ   r6  r   s   `   @@@r>   %test_dir_to_bucket_with_unicode_chars/TestRsync.test_dir_to_bucket_with_unicode_chars  s    
 !F""$J
I
 
8<  7=f '	#[13BKPCR 
 >3= 4= HrA   c                 \  ^ ^^^ T R                  5       mT R                  5       mSmT R                  TSSTS9  T R                  TSSTS9  T R                  TSSTS9  T R                  TS	S
TS9  T R                  TSSTS9  T R                  TSSTS9  T R                  TSSTS-
  S9  T R                  TSSTS-
  S9  T R                  TSSTS-
  S9  T R                  TS	STS9  T R                  TSSTS9  T R                  TSSTS-   S9  [	        [
        SSS9UUU U4S j5       nU" 5         g)z$Tests that rsync -u works correctly.r  r      obj1-1r   r      obj2-1r      obj3-1r   s   obj4-1r   s   obj5-1r  s   obj6-1   obj1-2r   rh  
   obj3-newers   obj4-2s   obj5-biggerrF   r   c            
      L  > TR                  SST[        T5      /5        TR                  STR                  S[        TS5      /SS95        TR                  STR                  S[        TS	5      /SS95        TR                  S
TR                  S[        TS5      /SS95        TR                  STR                  S[        TS5      /SS95        TR                  STR                  S[        TS5      /SS95        TR                  TR                  S[        T S-   5      5        g )Nr
   z-uobj1-1r#  r   Tr$  obj2-1r   obj3-1r   zobj4-1r   zobj5-biggerr   r  r   )r   rQ   r5   r   r   r   )
ORIG_MTIMEr   r;   r   s   r>   r  4TestRsync.test_dir_to_bucket_minus_u.<locals>._Checke  s1   
nngtVT*-=>?


..%j&!9:$.
OQ 

..%j&!9:$.
OQ 

..%j&!9:$.
OQ 

..%j&!9:$.
OQ 

..%j&!9:$.
OQ j44f!*q.13rA   Nr  )r;   r  rG  r   r   s   ` @@@r>   test_dir_to_bucket_minus_u$TestRsync.test_dir_to_bucket_minus_u!  s   !F""$JJ"((&  ( 	"((&  ( 	"((&  ( 	"((&  ( 	"((&  ( 	"((&  ( 	v"(!*(1n  .
 	v"(!*(1n  .
 	v"(!.(1n  . 	v"(!*(  *
 	v"(!/(  * 	v"(!*(1n  .
 >33 43. HrA   c                 h  ^ ^^ T R                  5       mT R                  5       mSnT R                  TSSS9  T R                  TSSS9  T R                  TSSUS	9  T R                  TSS
S9  T R                  TSSUS9  T R                  TSSUS-
  S9  [	        [
        SSS9UU U4S j5       nU" 5         g)z$Tests that rsync -i works correctly.r  r   r>  r   r   r?  r   r@  r   rA  r  rh  rB  r   rF   r   c            
      J  > TR                  SST[        T 5      /5        TR                  STR                  S[        T S5      /SS95        TR                  STR                  S[        T S	5      /SS95        TR                  S
TR                  S[        T S5      /SS95        g )Nr
   z-irD  r#  r   Tr$  rE  r   rF  r   )r   rQ   r5   )r   r;   r   s   r>   r  4TestRsync.test_dir_to_bucket_minus_i.<locals>._Check  s    
nngtVT*-=>?


..%j&!9:$.
OQ 

..%j&!9:$.
OQ 

..%j&!9:$.
OQrA   Nr  )r;   rG  r  r   r   s   `  @@r>   test_dir_to_bucket_minus_i$TestRsync.test_dir_to_bucket_minus_i  s    !F""$JJ"((  * 	"((  * 	"((&  ( 	v)Lv"(!*(  *
 	v"(!.(1n  .
 >3Q 4Q HrA   c                    Sn/ nU R                  5       nU R                  5       nSnU R                  XEU5      n[        S/5      nU R	                  SSU[        U5      /5        [        X@R                  U5      5      n[        [        U5      U R                  U5      5      n	U R                  [        U5      U5        U R                  [        U	5      U5        g)z7Test to ensure filenames with whitespace can be rsyncedzfoo bar baz.txtz6File from rsync test: test_rsync_files_with_whitespacez/foo bar baz.txtr
   r   NrN   rM   r  r8  r   rQ   r   rr  r   r5   r   
r;   r:  
local_urisrZ   r   r   
local_filer6  r   r   s
             r>    test_rsync_files_with_whitespace*TestRsync.test_rsync_files_with_whitespace  s     HJ""$J!FGH$$Vx@J%'9&:; 	NNGT64
+;<=v//78HtJ')<)<Z)HIHS]$9:S]$9:rA   c                    Sn/ nU R                  5       nU R                  5       nSnU R                  XEU5      n[        S/5      nU R	                  SSU[        U5      /5        [        X@R                  U5      5      n[        [        U5      U R                  U5      5      n	U R                  [        U5      U5        U R                  [        U	5      U5        g)z?Test to ensure filenames with special characters can be rsyncedu   Æ.txtz>File from rsync test: test_rsync_files_with_special_charactersu   /Æ.txtr
   r   NrQ  rR  s
             r>   (test_rsync_files_with_special_characters2TestRsync.test_rsync_files_with_special_characters  s    HJ""$J!FOH$$Vx@J%yk2 	NNGT64
+;<=v//78HtJ')<)<Z)HIHS]$9:S]$9:rA   z0No compressed transport encoding support for S3.z9No compressed transport encoding support for the XML API.c           
         Sn/ nU R                  5       nU R                  5       nSS-  nU H$  nUR                  U R                  XEU5      5        M&     U R	                  SSSSU[        U5      /SS	9nU R                  U[        U5      5        U H  nU R                  S
U-  U5        M     U R                  (       d  U R                  SU5        gg))Test gzip encoded files upload correctly.rl  test.txttest.xmlr  '  -Dr
   -Jr   Tr   2Using compressed transport encoding for file://%s.4send: Using gzip transport encoding for the request.N
rN   rM   appendr  r   rQ   r  lenr{   r   	r;   
file_namesrS  rZ   r   r   ri  r   	local_uris	            r>   &test_gzip_transport_encoded_all_upload0TestRsync.test_gzip_transport_encoded_all_upload  s     2JJ""$J!Fe|H	++FiHI   ^^	wdF	j		  F 	
C
O<	
mm
>)
L
   ##
mmJ $rA   c                 T   SnSn/ n/ nU R                  5       nU R                  5       nSS-  nU H$  nUR                  U R                  XgU5      5        M&     U H$  nUR                  U R                  XgU5      5        M&     U R	                  SSSSS	U[        U5      /S
S9n	U R                  U[        U5      [        U5      -   5        U H  n
U R                  SU
-  U	5        M     U H  n
U R                  SU
-  U	5        M     U R                  (       d  U R                  SU	5        gg)r[  )r]  z	photo.txt)fileztest.pngr^  r  r_  r`  r
   z-jtxtr   Tr   rb  rc  N)rN   rM   re  r  r   rQ   r  rf  r{   r   r   )r;   file_names_validfile_names_invalidlocal_uris_validlocal_uris_invalidrZ   r   r   ri  r   ri  s              r>   +test_gzip_transport_encoded_filtered_upload5TestRsync.test_gzip_transport_encoded_filtered_upload  sT    19""$J!Fe|H%	d11&INO &'	 3 3F4=!? @ ( ^^	weT6	j		  F 	
 #$4 5<N8O OQ &	
mm
>)
L
 & (	

>)
L
 ( ##
mmJ $rA   c                    Sn/ nU R                  5       nU R                  5       nSS-  nU H$  nUR                  U R                  XEU5      5        M&     U R	                  SSSSSU[        U5      /S	S
9nU R                  U[        U5      5        U H  nU R                  SU-  U5        M     U R                  (       d  U R                  SU5        gg)r[  r\  r  r_  r`  r  r
   ra  r   Tr   rb  rc  Nrd  rg  s	            r>   /test_gzip_transport_encoded_all_upload_parallel9TestRsync.test_gzip_transport_encoded_all_upload_parallel  s     2JJ""$J!Fe|H	++FiHI   ^^	tWdD&	j		  F 	
C
O<	
mm
>)
L
   ##
mmJ $rA   z$Test uses gs-specific KMS encryptionc                    U R                  5       n[        U5      S-   nSnSnU R                  5       nU R                  UUUS9  [	        5       n[        SSU4/5         U R                  SXR/5        S S S 5        [        S/5         U R                  S	S
U< SU< 3/SS9nS S S 5        U R                  WSU-  5        g ! , (       d  f       NS= f! , (       d  f       N5= f)Nz/foor  s   barr  rk   encryption_keyr
   rk   
prefer_apijsonr  z-Lr`  Tr$  zKMS key:\s+%s)rN   rQ   rM   r  r   r   r   r   )r;   rZ   cloud_container_suriobj_nameobj_contentstmp_dirkey_fqnstdouts           r>   $test_kms_key_applied_to_dest_objects.TestRsync.test_kms_key_applied_to_dest_objects5  s    ""$J
+f4HL  "Gw"*!-  / 12G 
*:GDE	F
nngw=> 
G 
?@	A~~"6A
B  f 
B 	V-78 
G	F 
B	As   B>C>
C
Cz>Tests that gs-specific encryption settings are skipped for s3.c                     U R                  5       nU R                  USS9  U R                  5       nSn[        S/5         U R	                  SU[        U5      /5        S S S 5        g ! , (       d  f       g = f)Nr  r   r   zGprojects/myproject/locations/global/keyRings/mykeyring/cryptoKeys/mykeyrz  r
   )rM   r  rN   r   r   rQ   )r;   r  rZ   	dummy_keys       r>   :test_kms_key_specified_will_not_prevent_non_kms_copy_to_s3DTestRsync.test_kms_key_specified_will_not_prevent_non_kms_copy_to_s3L  sk      "Gw8""$J$I
 
?@	A
nngwZ(89: 
B	A	As   A&&
A4c           
         U R                  5       nU R                  5       nU R                  USSS9  U R                  SSSS[        U5      [        U5      /SS	9nU R                  (       aR  [        [        R                  " S
U5      5      n[        [        R                  " SU5      5      nU R                  XE5        g [        R                  " SU5      nU R                  U5        U H  nU R                  SU5        M     g )Nr   r   r   z-DD-hzarbitrary:headerr
   Tr   z= request start =zb'arbitrary': b'header'zHeaders: \{([\s\S]*?)\}z'arbitrary': 'header')rN   r   r   rQ   r   rf  refindallr5   r  r{   )r;   r   r   r   request_counttarget_header_countheaders_for_all_requestsheaderss           r>   0test_bucket_to_bucket_includes_arbitrary_headers:TestRsync.test_bucket_to_bucket_includes_arbitrary_headersY  s    ##%K##%K"(&  ( ^^t'[[
 +/  0F "**%96BCm

+Ev NO
}:!#,F!O
oo./-'-w7 .rA   zTests that S3 SSE-C is handled.c           
      "   U R                  5       nU R                  USS9nU R                  5       nU R                  5       nSSSSR                  [        5      SSR                  [
        5      /n[        S/5         U R                  USU[        US	5      /-   5        U R                  US
[        U5      [        U5      /-   5        U R                  US[        US	5      /-   SS9nS S S 5        U R                  WS5        g ! , (       d  f       N!= f)Nr  r  r  z8"x-amz-server-side-encryption-customer-algorithm:AES256"z."x-amz-server-side-encryption-customer-key:{}"z2"x-amz-server-side-encryption-customer-key-md5:{}")rk   check_hashesneverr  rl  r
   r#  Tr$  r  )
rM   r  rN   r|   r   r   r   r   rQ   r5   )r;   r  tmp_filebucket_uri1bucket_uri2header_flagsr   s          r>   -test_s3_sse_is_handled_with_arbitrary_headers7TestRsync.test_s3_sse_is_handled_with_arbitrary_headersp  s      "G""'F"CH##%K##%K 	B8??"	$<CC&	(	L 
BC	D
nn\T8T+v5N$OOP
nn\tK0;')) * !&[&(A B!C.2   4h 
E 	Xu% 
E	Ds   ;A*D  
Dr]   N)T)Ir^   r_   r`   ra   rb   r   r   r   r   r   r   skipIfr)   
skipUnlessr   r   r  r  r5  rS  rd  r   r  r  r  r  r  r  r  r  r9  rE  rU  r]  r   rg  rn  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r  r  r"  r'  r*  r0  r;  rI  rN  rU  rX  r   rj  rs  rv  r  r   r  r  r  rf   r]   rA   r>   r   r      s   ,.(+ +/	H0H<| ??:KL+-/KLf M MfP>J@"HH[z}~hV !+-/KL^ M !^B !+-/KL=1 M !=1@ !+-/KLN M !N` +-/KLI MIV +-/KLC MCJ(T +-/KL
C M
C !+-/KLr M !rh ??:KLa MaF !??:KL+-/KLI1 M M !I1V +-/KL MB +-/KL% M%N +-/KL > ?? M: ??:JKD LD ??:JKO LOb 45/ 6/b D$L)VB(T,\..8t/b77C<KLAFQP ??:FHHB ??:JLLB\|-^;$;$ ?@IJ  ! K A6 ?@IJ # ! K A#J ?@IJ  ! K A4 349 59, MN
; O
;8. ./& 0&rA   r   )Mrb   
__future__r   r   r   r   r   r  r   r   r7   gslibr	   gslib.commandsr
   gslib.project_idr   gslib.storage_urlr   gslib.tests.testcaseteststestcase)gslib.tests.testcase.integration_testcaser   r   r   gslib.tests.utilr   r   r   r   r   rQ   r   r   r   r   r   r   r   r   r   gslib.utils.boto_utilr   gslib.utils.hashing_helperr    gslib.utils.posix_utilr!   r"   r#   r$   r%   r&   gslib.utils.retry_utilr'   gslib.utils.system_utilr(   r)   gslib.utilsr*   gslib.testsr+   r,   r-   r.   r/   PY3r  r   MACOS_WARNINGr   GsUtilUnitTestCaser2   ShimUnitTestBaserh   GsUtilIntegrationTestCaser   r]   rA   r>   <module>r     s3   + & %  ' 	 	  
    . 2 ' ' ? ? @ ? 3 7 , 0 * , < - , : 1 2 $ % 6 @ 9 + , - * + ( * . ! +**& 77	$Q  
5E/0* I  (4/*<*)H// )XOH55 OB^3&22 ^3&rA   