
    u_                       S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKJr  SSKJr  SSKrSS	KJr  SS
KJr  SSKJr  SSKJ r   SSK!J"r"  SSK!J#r#  SSK!J$r$  SSK%J&r&  SSK'J(r(  SSK)J*r*  SSK+J,r,  SSK-J.r.  SSK/J0r0  SSK1J2r2  SSK3J4r4  SSK5J6r6  SSK5J7r7  SSK8J9r9  SSK:J;r;  SSK<J=r=  SSK<J>r>  SSK<J?r?  SSK@JAs  JBrB  SSKCJDrD  SS KEJFrF  SS!KEJGrG  SS"KEJHrH  SS#KEJIrI  SS$KJJKrK  SS%KJJLrL  SS&KJJMrN  SS'KJJOrO  SS(KJJPrP  SS)KJJQrQ  SS*KJJRrR  SS+KJJSrS  SS,KJJTrU  SS-KJJVrV  SS.KJJWrW  SS/KJJXrX  SS0KJJYrY  SS1KJJZrZ  SS2KJJ[r[  SS3KJJ\r\  SS4KJJ]r]  SS5KJJ^r^  SS6KJJ_r_  SS7KJJ`r`  SS8KJJara  SS9KJJbrb  SS:KJJr  SS;KcJdre  SS<KfJgrg  SS=KfJhrh  SS>KfJiri  SS?KjJkrk  SS@KlJmrm  SSAKnJoro  SSBKpJqrq  SSCKpJrrr  SSDKsJtrt  SSEKsJuru  SSFKsJvrv  SSGKsJwrw  SSHKxJyry  SSIKxJzrz  SSJKxJ{r{  SSKK|J}r}  SSLK~Jr  SSMK~Jr  SSNK~Jr  SSOK~Jr  SSPK~Jr  SSQK~Jr  SSRK~Jr  SSSK~Jr  SSTKJr  SSUKJr  SSVKJr  SSWKJr  SSXKJr  SSYKJr  SSZKJr  SS[KJr  SS\KlJr  SSKrSS]KJr  SS^KJr  SS^KJr  \GR0                  (       a  \r\(       d*  SS_KJr  SS`KJJr  SSaKJJr  SSbKJJr  SScKJJr  SSdKJJr  SSeKJJr  SfrSzSg jrSzSh jrSzSi jrSj rSk r " Sl Sm\5      r " Sn So\5      r " Sp Sq\5      r " Sr Ss\5      r " St Su\BGR\                  5      r " Sv Sw\BGR`                  5      r " Sx Sy\BGRd                  5      rg){!Integration tests for cp command.    )absolute_import)division)print_function)unicode_literalsN)mock)
exceptions)storage_uri)ResumableTransferDisposition)StorageResponseError)BucketStorageUri)command)	exception)name_expansion)!ResumableUploadStartOverException)(DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD)'ShimTranslatePredefinedAclSubOptForCopy)ApiSelector)_DEFAULT_DOWNLOAD_CHUNK_SIZE)DiscardMessagesQueue)InvalidUrlError)
GcsJsonApi)ObjectFromTracker)WriteParallelUploadTrackerFile)PopulateProjectId)StorageUrlFromString)"EnsureRewriteResumeCallbackHandler)HaltingRewriteCallbackHandler)RewriteHaltException)NotParallelizable)	SkipForGS)	SkipForS3)
SkipForXML)SkipForJSON)"AuthorizeProjectToUseTestingKmsKey)BuildErrorRegex)GenerationFromURI)HaltingCopyCallbackHandler)#HaltOneComponentCopyCallbackHandler)HAS_GS_PORT)HAS_S3_CREDS)KmsTestingResources)ObjectToURI)ORPHANED_FILE)POSIX_GID_ERROR)POSIX_INSUFFICIENT_ACCESS_ERROR)POSIX_MODE_ERROR)POSIX_UID_ERROR)SequentialAndParallelTransfer)SetBotoConfigForTest)SetEnvironmentForTest)TailSet)TEST_ENCRYPTION_KEY1)TEST_ENCRYPTION_KEY1_SHA256_B64)TEST_ENCRYPTION_KEY2)TEST_ENCRYPTION_KEY3)unittest)storage_v1_messages)DeleteTrackerFile)GetRewriteTrackerFilePath)!GetSlicedDownloadTrackerFilePaths)BytesToFixedWidthString)hashing_helper)UsingCrcmodExtension)START_CALLBACK_PER_BYTES)UTF8)GetTrackerFilePath)PARALLEL_UPLOAD_STATIC_SALT)PARALLEL_UPLOAD_TEMP_NAMESPACE)TrackerFileType)"CalculateB64EncodedMd5FromContents)CalculateMd5FromContents)GetMd5)CreateCustomMetadata)GID_ATTR)	MODE_ATTR)NA_ID)NA_MODE)UID_ATTR)ParseAndSetPOSIXAttributes)ValidateFilePermissionAccess)ValidatePOSIXMode)Retry)
IS_WINDOWS)get_random_ascii_chars)	EIGHT_MIB)HumanReadableToBytes)MakeHumanReadable)ONE_KIB)ONE_MIB)	shim_util)http_client)range)util)DEFAULT_MODE)GetInvalidGid)GetNonPrimaryGid)GetPrimaryGid)INVALID_UID)USER_ID)   ERRORziAt most one of --gzip-in-flight | --gzip-in-flight-all | --gzip-local | --gzip-local-all can be specifiedc                 r   SnU R                   (       aD  [        R                  " S5      =pg[        R                  " S5      n[        R                  " S5      n	O@[        U[        5      n[        U[
        5      n[        U[        5      n	[        U[        5      n[        SXW0[        [        XX0[        [        [        SXX0[        [        XY0[        [        [        SXY0[        [        [        [        XY0[        [        [        [        [        SXY0[        [        [        [        XY0[        [        [        [        XY0[        [        [        [        [        SXY0[        [        [        [        [        SXY0[        [        [        [        XX0[        [        [        [        [        SXX0[        [        [        S	XV0S
.n
[         R"                  " U
5       GH  u  pU R%                  U5        UR'                  [        5      nUb  [)        U5      (       a  U" 5       nUR'                  [        5      nUb  [)        U5      (       a  U" 5       nUR'                  [        5      nU R+                  U R,                  UR.                  UR0                  UUUS9  U R3                  U(       a  SOSS[5        XR0                  5      U/SSS9nU R                   (       a  SnO[6        nU R9                  UUSU< SU< SU< 35        X   nU R;                  UR=                  U5      SU< SUR>                  < SU< 35        [A        [5        U5      U RC                  U5      5      n[A        X0RE                  U5      5      nU RG                  U[I        SUR0                  -  /5      5        U RG                  U[I        S/5      5        GM     g)aD  Helper function for preserve_posix_errors tests in test_cp and test_mv.

Args:
  cls: An instance of either TestCp or TestMv.
  bucket_uri: The uri of the bucket that the object is in.
  obj: The object to run the tests on.
  tmpdir: The local file path to cp to.
  is_cp: Whether or not the calling test suite is cp or mv.
error_regexz;User \d+ owns file, but owner does not have read permissionz2GID in .* metadata doesn't exist on current systemz2UID in .* metadata doesn't exist on current system333420530640240)test1test2test3test4test5test6test7test8test9test10test11test12test13test14Nuidgidmodecpmv-P   Texpected_statusreturn_stderrrh   zError during test "z": z not found in stderr:
zTest z: did not match expected error; could not find a match for z

in stderr:
z/%s )%_use_gcloud_storagerecompiler&   r0   r/   r2   r1   rN   rM   rb   rQ   re   rd   rc   rf   six	iteritemsClearPOSIXMetadatagetcallableSetPOSIXMetadatadefault_providerbucket_nameobject_name	RunGsUtilsurir.   assertIn
assertTruesearchpatternr6   FlatListBucketFlatListDirassertEqualset)cls
bucket_uriobjtmpdiris_cp	error_keyinsufficient_access_errorno_read_access_errormissing_gid_errormissing_uid_errortest_params	test_name
attrs_dictr   r   r   stderrgeneral_posix_errorrj   listing1listing2s                        &platform/gsutil/gslib/tests/test_cp.py TestCpMvPOSIXBucketToLocalErrorsr      sl    )79zzF8H H

=?

=? !0,!.'_='_=*30@A U

 M

 M
U
 K

 K
U
 K
M
 K
M
U
	 K
M
 K
$
 K
M
U
	 K
$
U
	 G
M
 G
M
U
	 M
U
GH+V  #}}[9i3 ..
"C
8C==Ec
..
"C
8C==Ec>>)$D--#//  "  $ ]]4Z)6 ,-)-  /F #)LLV	'	12
 'KNN6""+[-@-@&	JK tJ'););J)GHHvv67HOOHc53??#:";<=OOHc2$i(_  :    c                    [         R                  " U5      R                  n[        R                  " 5       n[
        U0[
        U0[
        U[        S0[
        U[        S0[        [        0[        [        [        S0[        [        [
        U0[        [        [
        U0[        [        [
        U[        S0[        [        [
        U[        S0S.
n[        R                  " U5       Hk  u  pxUR                  [        5      n	UR                  [
        5      n
UR                  [        5      nU R                  UUUR                  [        5      U	U
US9  Mm     [        R                  " U5       H*  nU R!                  U(       a  SOS	S
[#        X5      U/5        M,     [%        X R'                  U5      5      nU R)                  U[+        / SQ5      5        U R-                  [         R.                  R1                  US5      U[2        S9  U R-                  [         R.                  R1                  US5      U[2        S9  U R-                  [         R.                  R1                  US5      USS9  U R-                  [         R.                  R1                  US5      USS9  U R-                  [         R.                  R1                  US5      [        U[2        S9  U R-                  [         R.                  R1                  US5      [        USS9  U R-                  [         R.                  R1                  US5      [        U[2        S9  U R-                  [         R.                  R1                  US5      [        U[2        S9  U R-                  [         R.                  R1                  US5      [        USS9  U R-                  [         R.                  R1                  US5      [        USS9  g)a   Helper function for preserve_posix_no_errors tests in test_cp and test_mv.

Args:
  cls: An instance of either TestCp or TestMv.
  bucket_uri: The uri of the bucket that the object is in.
  tmpdir: The local file path to cp to.
  is_cp: Whether or not the calling test suite is cp or mv.
440444rl   433442
obj1obj2obj3obj4obj5obj6obj7obj8obj9obj10)r   r   contentsr   r   r   r   r   r   )
z/obj1z/obj2z/obj3z/obj4z/obj5z/obj6z/obj7z/obj8z/obj9z/obj10r   )r   r   r   r   i   r   i$  r   r~   r   i  r   r   r   i  r   i"  N)osstatst_gidr`   rc   rM   rN   rQ   rf   r   r   r   CreateObjectencoderD   iterkeysr   r   r6   r   r   r   VerifyLocalPOSIXPermissionspathjoinra   )r   r   r   r   primary_gidnon_primary_gidr   obj_namer   r   r   r   listings                r   "TestCpMvPOSIXBucketToLocalNoErrorsr   7  sv    &&+))+/ K O K
U
 O
U
 G G
U
 G
K
 G
O
 G
K
U G
O
UG(+R "mmK8h
..
"C
..
"C>>)$D
!)&ood3   	 9 ,,{+hMMD$	j	#V	-. ,
 FOOF34'//	  	
 !!"'',,vv">&1'3 " 5 !!"'',,vv">&5'3 " 5 !!"'',,vv">&1', " . !!"'',,vv">&5', " . !!"'',,vv">&-&1'3 " 5 !!"'',,vv">&-&1', " . !!"'',,vv">&-&1'3 " 5 !!"'',,vv">&-&5'3 " 5 !!"'',,vv">&-&1', " . !!"'',,vw"?&-&5', " .r   c                    [         R                  " 5       n[        R                  " 5       n[        U0[        U0[        U[
        S0[        U[
        S0[        [        0[        [        [
        S0[        [        [        U0[        [        [        U0[        [        [        U[
        S0[        [        [        U[
        S0S.
n[        R                  " U5       GHk  u  pgUR                  [        [        5      nUR                  [        [        5      n	UR                  [
        [        5      n
U
[        :w  a  [        [        U
S5      5        [        UU[        U	5      [        U
5      S9  U R!                  S	XU
S
9nU R#                  U(       a  SOSSU[%        X5      /5        U[        :w  a+  U R'                  UR(                  U[        [+        U5      5        U	[        :w  a+  U R'                  UR(                  U[        [+        U	5      5        U
[        :w  d  GM@  U R'                  UR(                  U[
        [+        U
5      5        GMn     g)zHelper function for testing local to bucket POSIX preservation.

Args:
  cls: An instance of either TestCp or TestMv.
  bucket_uri: The uri of the bucket to cp/mv to.
  is_cp: Whether or not the calling test suite is cp or mv.
r   r   rl   r   r   r      r~      foo)r   r   r   r   r   r   r   N)r   getgidr`   rc   rM   rN   rQ   rf   r   r   r   rO   rP   rT   intrS   CreateTempFiler   r   VerifyObjectCustomAttributer   str)r   r   r   r   r   r   r   r   r   r   r   fpaths               r   "TestCpMvPOSIXLocalToBucketNoErrorsr     s    		+))+/ K O K
U
 O
U
 G G
U
 G
K
 G
O
 G
K
U G
O
UG(+R "mmK8h
..5
)C
..5
)C>>)W-DwD!% %(%(X&)$i1 CtLEMMD$	j	#	%& e|	%%j&<&<h&.C:
e|	%%j&<&<h&.C:w	%%j&<&<h&/T<+ 9r   c                     [        U S5       nUR                  UR                  5       5        S S S 5        g ! , (       d  f       g = f)Nrb)openappendread)	fifo_pathlist_for_outputfs      r   _ReadContentsFromFifor     s.    It1668$ s	    6
Ac                 r    [        US5       nUR                  U 5        S S S 5        g ! , (       d  f       g = f)Nwb)r   write)r   r   r   s      r   _WriteContentsToFifor     s&    ItGGH s   (
6c                   $    \ rS rSrSrS rS rSrg)&_JSONForceHTTPErrorCopyCallbackHandleri  zDTest callback handler that raises an arbitrary HTTP error exception.c                 *    Xl         X l        SU l        g NF)_startover_at_byte_http_error_numstarted_over_once)selfstartover_at_bytehttp_error_nums      r   __init__/_JSONForceHTTPErrorCopyCallbackHandler.__init__   s    /)"Dr   c                 H   XR                   :  a  U R                  (       d  [        R                  R	                  SU R
                  < SU R                   < S[        U5      < S[        U5      < S3	5        SU l        [        R                  " SU R
                  0SS5      egg)	<Forcibly exits if the transfer has passed the halting point.zForcing HTTP error z after byte . / transferred.
TstatusN)	r   r   sysr   r   r   rZ   apitools_exceptions	HttpErrorr   total_bytes_transferred
total_sizes      r   call+_JSONForceHTTPErrorCopyCallbackHandler.call  s    #:#::""	jj,,d.E.E)*AB)*5	7 8
  $d))8T5I5I*J*.6 6 # 	;r   )r   r   r   N__name__
__module____qualname____firstlineno____doc__r   r  __static_attributes__ r   r   r   r     s    L#6r   r   c                   $    \ rS rSrSrS rS rSrg)/_XMLResumableUploadStartOverCopyCallbackHandleri  zETest callback handler that raises start-over exception during upload.c                     Xl         SU l        g r   r   r   )r   r   s     r   r   8_XMLResumableUploadStartOverCopyCallbackHandler.__init__  s    /"Dr   c           
      @   XR                   :  a  U R                  (       d}  [        R                  R	                  SU R                   < S[        U5      < S[        U5      < S35        SU l        [        R                  R                  S[        R                  5      egg)r   4Forcing ResumableUpload start over error after byte r   r   r   TzForcing upload start overN)r   r   r   r   r   rZ   botor   ResumableUploadExceptionr   
START_OVERr   s      r   r  4_XMLResumableUploadStartOverCopyCallbackHandler.call  s    #:#::""	jj ""$56M$NZ(*+
  $dNN33
%'C'N'NP P # 	;r   r  Nr  r  r   r   r  r    s    M#
Pr   r  c                   $    \ rS rSrSrS rS rSrg)-_DeleteBucketThenStartOverCopyCallbackHandleri*  zATest callback handler that deletes bucket then raises start-over.c                 *    Xl         X l        SU l        g r   )r   _bucket_urir   )r   r   r   s      r   r   6_DeleteBucketThenStartOverCopyCallbackHandler.__init__-  s    /!"Dr   c           
        ^  UT R                   :  a  T R                  (       d  [        R                  R	                  ST R
                  R                  -  5        [        [        SSS9U 4S j5       nU" 5         [        R                  R	                  ST R                   < S[        U5      < S[        U5      < S	35        S
T l        [        S5      egg)r   zDeleting bucket (%s)   r   triestimeout_secsc                    > [        TR                  R                  SS95      n U  H?  nTR                  R                  5       R	                  UR
                  UR                  S9  MA     TR                  R                  5         g )NT)all_versions)
version_id)listr  list_bucket
get_bucket
delete_keynamer#  delete_bucket)bucket_listkr   s     r   DeleteBucketH_DeleteBucketThenStartOverCopyCallbackHandler.call.<locals>.DeleteBucket9  so    4++77T7JKA



%
%
'
2
2166>?ll 3 L  	&&(r   r  r   r   r   TzArtificially forcing start-overN)r   r   r   r   r   r  r   rU   r   rZ   r   )r   r   r  r,  s   `   r   r  2_DeleteBucketThenStartOverCopyCallbackHandler.call3  s    4#:#::""	jj-1A1A1M1MNO!;) <) n	jj ""$56M$NZ(*+
  $d-.OPP% # 	;r   )r  r   r   Nr  r  r   r   r  r  *  s    I#Qr   r  c                   *    \ rS rSrSr SS jrS rSrg)_ResumableUploadRetryHandleriK  zFTest callback handler for causing retries during a resumable transfer.c                 B    Xl         X l        X0l        X@l        SU l        g )Nr   )_retry_at_byte_exception_to_raise_exception_args_num_retries_retries_made)r   retry_at_byteexception_to_raiseexc_argsnum_retriess        r   r   %_ResumableUploadRetryHandler.__init__N  s$    
 (1##Dr   c                     XR                   :  aI  U R                  U R                  :  a.  U =R                  S-  sl        U R                  " U R                  6 egg)z(Cause a single retry at the retry point.r   N)r2  r6  r5  r3  r4  )r   r   unused_total_sizes      r   r  !_ResumableUploadRetryHandler.call[  sT    #6#66T...
A$$d&:&:;; 	/ 	7r   )r4  r3  r5  r6  r2  N)r   r  r  r   r   r0  r0  K  s    N 	
<r   r0  c                   \   \ rS rSrSr\S-  rS rS r\	S 5       r
\	S 5       rS rS	 r\	\" S
5      S 5       5       r\	\" S
5      S 5       5       r\	\" S
5      \R$                  " \S5      S 5       5       5       r\R$                  " \S5      \	S 5       5       r\R$                  " \S5      \	S 5       5       r\R$                  " \S5      \	S 5       5       r\	S 5       r\R$                  " \S5      \	S 5       5       r\R$                  " \S5      \	S 5       5       r\R$                  " \S5      \	S 5       5       rS r\	S 5       rS rS r\R$                  " \S5      \	S 5       5       r \	S 5       r!\	S 5       r"\	S 5       r#\	S  5       r$\	\" S!5      S" 5       5       r%\	\&" S#5      S$ 5       5       r'\	S% 5       r(S& r)\*" S'5      S( 5       r+\	\*" S)5      S* 5       5       r,\	\*" S)5      S+ 5       5       r-\	S, 5       r.\	\*" S'5      S- 5       5       r/S. r0\	S/ 5       r1S0 r2\Rf                  " \4S15      S2 5       r5\Rf                  " \4S15      \Rl                  " S35      S4 5       5       r7\Rf                  " \4S15      S5 5       r8\Rl                  " S65      S7 5       r9S8 r:\Rf                  " \;(       + S95      S: 5       r<S; r=S< r>S= r?S> r@\	S? 5       rAS@ rBSA rC\	SB 5       rDSC rESD rFSE rG\H" SF5      SG 5       rISH rJ\Rf                  " \4S15      SI 5       rK\*" SJ5      SK 5       rLSL rMSM rNSN rO\R$                  " \SO5      \	SP 5       5       rP\	SQ 5       rQSR rR\	SS 5       rS\	ST 5       rT\	SU 5       rU\*" SV5      \" SW5      \	SX 5       5       5       rV\*" SV5      \" SW5      \	SY 5       5       5       rW\*" SV5      \" SW5      \	SZ 5       5       5       rX\	S[ 5       rY\*" SV5      \" SW5      \	S\ 5       5       5       rZS] r[S^ r\S_ r]S` r^Sa r_\	Sb 5       r`Sc ra\R$                  " \S5      Sd 5       rb\R$                  " \S5      Se 5       rcSf rd\	Sg 5       re\*" Sh5      \	Si 5       5       rf\*" Sh5      \	Sj 5       5       rg\*" Sh5      \	Sk 5       5       rh\*" Sl5      Sm 5       ri\*" Sl5      Sn 5       rj\*" Sl5      So 5       rkSp rl\*" Sq5      Sr 5       rm\*" SV5      \" SW5      \	Ss 5       5       5       rn\*" Sq5      St 5       ro\*" Sq5      Su 5       rp\*" Sv5      Sw 5       rq\*" Sq5      Sx 5       rr\*" Sq5      Sy 5       rs\*" Sq5      Sz 5       rt\*" Sq5      S{ 5       ru\*" Sq5      S| 5       rv\*" Sq5      S} 5       rw\*" Sq5      S~ 5       rx\*" Sq5      S 5       ry\*" S5      S 5       rz\*" S5      S 5       r{\|\*" Sq5      \R$                  " \S5      \	S 5       5       5       5       r}\|\R$                  " \S5      \	S 5       5       5       r~ SS jrS r\*" Sh5      S 5       r\*" Sh5      S 5       r\	S 5       r\Rf                  " \" 5       S5      \*" S5      S 5       5       rS rS rS rS rS r\*" SV5      \" SW5      \	S 5       5       5       r\R$                  " \S5      \	S 5       5       rS rS r\	S 5       r\*" Sq5      S 5       r\*" S5      S 5       r\Rf                  " \" 5       S5      \*" S5      S 5       5       r\Rf                  " \" 5       S5      \*" S5      S 5       5       r\Rf                  " \" 5       S5      \*" S5      S 5       5       r\Rf                  " \" 5       S5      \*" S5      S 5       5       r\Rf                  " \" 5       S5      \*" S5      S 5       5       r\Rf                  " \" 5       S5      \*" S5      S 5       5       r\*" Sq5      S 5       rS rS rS rS rS rS r\R$                  " \S5      \Rf                  " \" 5       S5      S 5       5       r\R$                  " \S5      S 5       r\R$                  " \S5      S 5       rS r\" S5      S 5       r\*" S5      S 5       r\*" S5      S 5       r\*" S5      S 5       r\*" S5      S 5       r\*" S5      S 5       r\*" S5      \" S5      S 5       5       r\Rf                  " \S5      S 5       rS r\	S 5       r\	S 5       rSrg)TestCpic  r   rg   c                 T    [         R                  " SSU-  5      nU R                  XS9$ )Ngslibztests/test_data/%s	file_namer   )pkgutilget_datar   )r   r(  r   s      r   _get_test_fileTestCp._get_test_filej  s.    )=)DEHAAr   c                     S/nUR                  U5        UR                  U5        UR                  U R                  " U40 UD65        g )Nr   )extendr   r   )r   src_path_tupledst_pathlist_for_return_valuekwargsarg_lists         r   )_CpWithFifoViaGsUtilAndAppendOutputToList0TestCp._CpWithFifoViaGsUtilAndAppendOutputToListn  sA     vHOON#OOH   !CF!CDr   c                 x   U R                  SS9nU R                  SS9nU R                  SSU[        U5      /SS9nU R	                  USR                  [        R                  " [        U5      5      5      5        U R                  UR                  5       S5        U R                  SS[        U5      U/SS9n[        US	5       nU R	                  USR                  [        R                  " [        U5      5      5      5        U R                  UR                  5       S5        S S S 5        g ! , (       d  f       g = f)
Nr   r      barr   -nTr   Skipping.*: {}r   r   r   r   r   assertRegexformatr   escaper   get_contents_as_stringr   r   r   key_urir   r   r   s        r   test_noclobberTestCp.test_noclobberx  s   0G0E^^	tUDM*$  @FV.55biiW6NOPW335v>^^T4g>*.  0F	eT	a
v077		$q'8JKL
qvvx( 
		   AD++
D9c                 x   U R                  SS9nU R                  SS9nU R                  SSU[        U5      /SS9nU R	                  USR                  [        R                  " [        U5      5      5      5        U R                  UR                  5       S5        U R                  SS[        U5      U/SS9n[        US	5       nU R	                  USR                  [        R                  " [        U5      5      5      5        U R                  UR                  5       S5        S S S 5        g ! , (       d  f       g = f)
Nr   rS  s   quuxr   rU  TrV  rW  r   rX  r]  s        r   test_noclobber_different_size$TestCp.test_noclobber_different_size  s   0G1E^^	tUDM*$  @FV.55biiW6NOPW335v>^^T4g>*.  0F	eT	a
v077		$q'8JKL
qvvx) 
		ra  c                    ^ ^^ T R                  SS9mT R                  < ST R                  < 3m[        [        SSS9UUU 4S j5       nU" 5         g )Nr   rS  z://   r   r  c                     > TR                  STT/SSS9n TR                  (       a  TR                  SU 5        g TR                  SU 5        g )Nr   r   Tr   znot found: 404zdoes not existr   r   r   )r   r   invalid_bucket_urir   s    r   _Check1TestCp.test_dest_bucket_not_exist.<locals>._Check  sO    ~~tU,>?./,0  2f 
	!	!&/&/r   )r   r   nonexistent_bucket_namerU   AssertionError)r   rj  r   ri  s   ` @@r   test_dest_bucket_not_exist!TestCp.test_dest_bucket_not_exist  sT    0E00$2N2NP >30 40 Hr   c           	         U R                  5       nU R                  5       nU R                  USS9nU R                  S[        U5      [        U5      /SS9nU R	                  UR                  S5      S5        U R                  UR                  S5      S5        U R                  SS	[        U5      [        U5      /SS9nU R                  US
R                  [        UUR                  5      5      5        g )Nr   r   r   r   TrV  Copyingr   rg   rU  rW  )
CreateBucketr   r   r   assertGreaterEqualcountassertLessEqualrY  rZ  r   )r   bucket1_uribucket2_urir^  r   s        r   test_copy_in_cloud_noclobber#TestCp.test_copy_in_cloud_noclobber  s    ##%K##%K;HG^^	tG}d;/0  FF 	FLL3Q7i0!4^^	tT']	k		+/  1F 	!((k.5.A.A*C DEr   z5Boto library does not handle objects with .. in them.c           	         U R                  5       nU R                  USSSS9nU R                  USSS9  U R                  5       nU R                  SS[	        U5      U/SS	9nU R
                  R                  UR                  UR                  5        U R                  S
[	        U5      < S[        R                  R                  U5      < S3U5        U R                  [        R                  R                  [        R                  R                  US5      5      5        U R!                  [        R                  R                  [        R                  R                  X1R                  S5      5      5        g )Ndir/../../../file   dataTr   r   r   prefer_json_apifile2r   r   r   r   -rrV  Skipping copy of source URL H because it would be copied outside the expected destination directory: .file)rs  r   CreateTempDirr   r   json_apiDeleteObjectr   r   r   r   r   abspathassertFalseexistsr   r   )r   r   r^  	directoryr   s        r   5test_skip_object_with_parent_directory_symbol_in_name<TestCp.test_skip_object_with_parent_directory_symbol_in_name  s6    ""$J:,?)004   6G 	")&  ( ""$I^^	tT*%y1  GF 	MMz55w7J7JKMM 
g	2	45;= 	RWW^^BGGLLF$CDEOO
rww||I/E/E$+- 	./r   c           	      v   U R                  5       nU R                  USSSS9nU R                  5       n[        R                  R                  US5      nU R                  SSSU[        U5      U/SS	9nU R                  R                  UR                  UR                  5        U R                  S
[        U5      < S[        R                  R                  U5      < S3U5        U R                  [        R                  R                  [        R                  R                  US5      5      5        [!        US5       nUR#                  5       nUS   R%                  5       R'                  S5      nU R)                  US   [        U5      5        U R)                  US   S5        S S S 5        g ! , (       d  f       g = f)Nr|  r}  Tr~  zlog.csvr   r  -LrV  r  r  r  r  rr   ,r   r   skip)rs  r   r  r   r   r   r   r   r  r  r   r   r   r  r  r  r   	readlinesstripsplitr   )	r   r   r^  r  log_pathr   r   linesresultss	            r   Btest_skip_parent_directory_symbol_in_name_is_reflected_in_manifestITestCp.test_skip_parent_directory_symbol_in_name_is_reflected_in_manifest  sh    ""$J:,?)004   6G ""$Iww||Iy1H^^	tT8	j	9	&  F 	MMz55w7J7JKMM 
g	2	45;= 	RWW^^BGGLLF$CDE	h	kkmea &&s+g
wqz4=1
wqz6*	 
		s   ;A&F**
F8z)os.symlink() is not available on Windows.c           	         U R                  5       nU R                  USSSS9nU R                  USSS9nU R                  5       n[        R                  R                  US5      n[        R                  R                  US5      n[        R                  " U5        [        R                  " Xe5        U R                  S	S
S[        U5      [        U5      U/SS9nU R                  R                  UR                  UR                  5        U R                  S[        U5      < SU< S3U5        U R                  [        R                  R!                  [        R                  R                  US5      5      5        U R#                  [        R                  R!                  [        R                  R                  US5      5      5        g )Nr|  r}  Tr~  r  r  linked_destinationdestination-Dr   r  rV  r  r  r  r  )rs  r   r  r   r   r   mkdirsymlinkr   r   r  r  r   r   r   r  r  r   )r   r   r^  second_key_urir  r  r  r   s           r   Atest_skip_parent_directory_symbol_object_with_symlink_destinationHTestCp.test_skip_parent_directory_symbol_object_with_symlink_destination  sr    ""$J:,?)004   6G &&*3:07 ' 9N ""$Ii1EF'',,y-8KHH[JJ{/^^dDZ^0
 +/  0F 	MMz55w7J7JKMM 
g*	,-35 	RWW^^BGGLL1CV$LMNOOBGGNN277<<0BG#LMNr   z#os.mkfifo not available on Windows.c                    SnU R                  5       nU R                  US9n/ n[        R                  " [        X$4S9nUR                  5         [        R                  " U R                  U4U/ 4S9nUR                  5         UR                  S5        UR                  S5        U(       d  U R                  S5        U R                  US   R                  5       U5        g )NrT  rS  targetargsx   &Reading/writing to the fifo timed out.r   )CreateTempFifor   	threadingThreadr   startrP  r   failr   r  )r   r   r   	file_pathr   read_threadwrite_threads          r   test_cp_from_local_file_to_fifo&TestCp.test_cp_from_local_file_to_fifo  s     H##%I##X#6IO""*?)2(DFK##==lIr*,L cS
ii89_Q'--/:r   c                    U R                  5       nU R                  5       nSnU R                  X#S9n/ n[        R                  " [
        X4S9nUR                  5         [        R                  " U R                  [        U5      4U/ 4S9nUR                  5         UR                  S5        UR                  S5        U(       d  U R                  S5        U R                  US   R                  5       U5        g )NrT  rq  r  r  r  r   )r  rs  r   r  r  r   r  rP  r   r   r  r   r  )r   r   r   r   obj_urir   r  r  s           r   test_cp_from_one_object_to_fifo&TestCp.test_cp_from_one_object_to_fifo1  s     ##%I""$JH:IGO""*?)2(DFK##==G}	2.0L cS
ii89_Q'--/:r   c                 0   U R                  5       nU R                  5       nSnSnU R                  X#S9nU R                  X$S9n/ n[        R                  " [
        X4S9nUR                  5         [        R                  " U R                  [        U5      [        U5      4U/ 4S9n	U	R                  5         U	R                  S5        UR                  S5        U(       d  U R                  S5        U R                  X7S   5        U R                  XGS   5        g )Ns   foo and bars   baz and quxrq  r  r  r  r   )r  rs  r   r  r  r   r  rP  r   r   r  r   )
r   r   r   	contents1	contents2obj1_uriobj2_urir   r  r  s
             r   %test_cp_from_multiple_objects_to_fifo,TestCp.test_cp_from_multiple_objects_to_fifoG  s    ##%I""$JII  J KH  J KHO""*?)2(DFK##==H~tH~.	2>@L cS
ii89MM)Q/0MM)Q/0r   c           	      P   U R                  5       nU R                  SSS[        US5      -  /SSS9nU R                  (       a   U R	                  S[        US5      -   U5        OU R	                  S	U5        U R                  US5      nU R                  UR                  5       S
5        g )Nr   -z%sfoobarTstdinr   zCopying file://- to zCopying from <STDIN>rT  )rs  r   r   r   r   StorageUriCloneReplaceNamer   r\  )r   r   r   r^  s       r   test_streamingTestCp.test_streaming`  s    ""$J^^	sD4
E223  F 
mm*T*e-DDfM
mm*F3--j%@GW335v>r   c           	         U R                  5       nU R                  5       nSnSn/ n[        R                  " [        XB4S9nUR                  5         [        R                  " U R                  U4[        X5      U4SS0S9nUR                  5         UR                  S5        UR                  S5        U(       d  U R                  S5        U R                  (       a/  U R                  S	R                  U[        X5      5      US
   5        OU R                  SUS
   5        U R                  X5      nU R                  UR                  5       U5        g )Nr  rT  r  r   Tr  r  rN  r  r  zCopying file://{} to {}r   zCopying from named pipe)rs  r  r  r  r   r  rP  r   r   r  r   r   rZ  r  r   r\  )	r   r   r   r   object_contentsr   r  r  r^  s	            r   "test_streaming_from_fifo_to_object)TestCp.test_streaming_from_fifo_to_objectn  s:    ""$J##%IKOO ##+?*9)EGL ""==lD9?K&(K Sc
ii89
mm
#
*
*9+/
+HJ
!

 mm-q/AB--jFGW335Gr   c                    U R                  5       nSn/ n[        R                  " [        X!4S9nUR	                  5         [        R                  " U R
                  U4SU4SS0S9nUR	                  5         UR                  S5        UR                  S5        U(       d  U R                  S5        U R                  US	   R                  5       R                  S
5      U5        g )NrT  r  r  return_stdoutTr  r  r  r   ascii)r  r  r  r   r  rP  r   r  r   r  r   )r   r   r   r   r  r  s         r   "test_streaming_from_fifo_to_stdout)TestCp.test_streaming_from_fifo_to_stdout  s     ##%IHO##+?*2)>@L""==lC1&(K Sc
ii89_Q'--/66w?Jr   c                    U R                  5       nSn/ n/ n[        R                  " [        X4S9nUR	                  5         [        R                  " U R
                  SX4SUS.S9nUR	                  5         UR                  S5        UR                  S5        U(       d  U R                  S5        U R                  US	   R                  5       U5        g )
NrT  r  )r  T)r   r  r  r  r  r   )
r  r  r  r   r  rP  r   r  r   r  )r   r   r   r   list_for_gsutil_outputr  r  s          r   "test_streaming_from_stdout_to_fifo)TestCp.test_streaming_from_stdout_to_fifo  s     ##%IHO""*?)2(DFK##==i8!
L cS
ii89_Q'--/:r   c                     U R                  5       nU R                  SSS[        U5      /SSSS9nU R                  (       a  U R	                  SU5        g U R	                  SU5        g )	Nr   r  r  Tr   )r  r   r   zDMultiple URL strings are not supported when transferring from stdin.z5Multiple URL strings are not supported with streamingrs  r   r   r   r   r   r   r   s      r   !test_streaming_multiple_arguments(TestCp.test_streaming_multiple_arguments  so    ""$J^^T3T*-=>"'*.,-  /F 
mm " mmKr   c                 L  ^ ^ T R                  5       n[        US5      mT R                  ST R                  S5      T/5        [	        [
        SSS9UU 4S j5       nU" 5         T R                  ST R                  S5      T/5        [	        [
        SSS9UU 4S	 j5       nU" 5         g
)z&Tests local detection of content type.r  r   test.mp3rf  r   r  c                     > TR                  SST/SS9n [        (       aE  TR                  [        R                  " SU 5      =(       d    [        R                  " SU 5      5        g TR                  U S5        g )Nlsr  Tr  zContent-Type:\s+audio/x-mpgzContent-Type:\s+audio/mpeg)r   rV   r   r   r   rY  stdoutdsturir   s    r   _Check10TestCp.test_detect_content_type.<locals>._Check1  sc    ~~tT62$~Gf	II4f= =II3V<	> 	!>?r   test.gifc                  P   > TR                  SST/SS9n TR                  U S5        g Nr  r  Tr  Content-Type:\s+image/gifr   rY  r  s    r   _Check20TestCp.test_detect_content_type.<locals>._Check2  -    ~~tT62$~Gf
v;<r   Nrs  r   r   rG  rU   rm  r   r   r  r  r  s   `   @r   test_detect_content_typeTestCp.test_detect_content_type  s     ""$J*e$FNND$--j96BC >3@ 4@ INND$--j96BC >3= 4= Ir   c                 T  ^ ^ T R                  5       n[        US5      mT R                  SSST R                  S5      T/5        [	        [
        SSS9UU 4S	 j5       nU" 5         T R                  SSST R                  S
5      T/5        [	        [
        SSS9UU 4S j5       nU" 5         g)z5Tests overriding content type with the default value.r  -hzContent-Type:r   r  rf  r   r  c                  P   > TR                  SST/SS9n TR                  U S5        g Nr  r  Tr  z(Content-Type:\s+application/octet-streamr  r  s    r   r  :TestCp.test_content_type_override_default.<locals>._Check1  -    ~~tT62$~Gf
vJKr   r  c                  P   > TR                  SST/SS9n TR                  U S5        g r  r  r  s    r   r  :TestCp.test_content_type_override_default.<locals>._Check2
  r  r   Nr  r  s   `   @r   "test_content_type_override_default)TestCp.test_content_type_override_default  s    ""$J*e$FNN				Z	(&	23
 >3L 4L INN				Z	(&	23
 >3L 4L Ir   c                 T  ^ ^ T R                  5       n[        US5      mT R                  SSST R                  S5      T/5        [	        [
        SSS9UU 4S	 j5       nU" 5         T R                  SSST R                  S
5      T/5        [	        [
        SSS9UU 4S j5       nU" 5         g)z+Tests overriding content type with a value.r  r  zContent-Type:text/plainr   r  rf  r   r  c                  P   > TR                  SST/SS9n TR                  U S5        g Nr  r  Tr  Content-Type:\s+text/plainr  r  s    r   r  2TestCp.test_content_type_override.<locals>._Check1  -    ~~tT62$~Gf
v<=r   r  c                  P   > TR                  SST/SS9n TR                  U S5        g r  r  r  s    r   r  2TestCp.test_content_type_override.<locals>._Check2)  r  r   Nr  r  s   `   @r   test_content_type_override!TestCp.test_content_type_override  s    ""$J*e$FNN'J'  >3> 4> INN'J'  >3> 4> Ir   z&magicfile is not available on Windows.c                    ^ ^ T R                  5       n[        US5      mT R                  SS9nT R                  SUT/5        [	        [
        SSS9UU 4S j5       nU" 5         g	)
z1Tests content type override with magicfile value.r     foo/bar
rS  r   rf  r   r  c                     > TR                  SST/SS9n [        R                  R                  SSS5      nU(       a  SOS	nTR	                  U S
U-  5        g )Nr  r  Tr  GSUtiluse_magicfileFz
text/plainzapplication/octet-streamzContent-Type:\s+%s)r   r  configgetboolrY  )r  r  content_typer  r   s      r   r  /TestCp.test_magicfile_override.<locals>._Check1:  sZ    ~~tT62$~Gfkk))(OUKm& #,F 
v4|CDr   N)rs  r   r   r   rU   rm  )r   r   r   r  r  s   `   @r   test_magicfile_overrideTestCp.test_magicfile_override0  sk     ""$J*e$F6ENND%() >3E 4E Ir   c                   ^ ^ T R                  5       n[        US5      mT R                  SS9nT R                  SSST R	                  S5      T/5        [        [        SS	S
9UU 4S j5       nU" 5         T R                  SSST R	                  S5      T/5        [        [        SS	S
9UU 4S j5       nU" 5         T R                  SSSUT/5        [        [        SS	S
9UU 4S j5       nU" 5         g)zCTests overriding content type when it does not match the file type.r  r  rS  r  Content-Type:image/gifr   r  rf  r   r  c                  P   > TR                  SST/SS9n TR                  U S5        g r  r  r  s    r   r  4TestCp.test_content_type_mismatches.<locals>._Check1Q  r  r   r  c                  P   > TR                  SST/SS9n TR                  U S5        g r  r  r  s    r   r  4TestCp.test_content_type_mismatches.<locals>._Check2^  r  r   c                  P   > TR                  SST/SS9n TR                  U S5        g r  r  r  s    r   _Check34TestCp.test_content_type_mismatches.<locals>._Check3h  r  r   N)rs  r   r   r   rG  rU   rm  )r   r   r   r  r  r  r  s   `     @r   test_content_type_mismatches#TestCp.test_content_type_mismatchesD  s    ""$J*e$F6ENN&J'  >3= 4= INN&J'  >3= 4= INND2D%HI >3= 4= Ir   c           	      >  ^ ^ T R                  5       n[        US5      mT R                  S5      nT R                  SSSUT/5        [	        [
        SSS9UU 4S	 j5       nU" 5         T R                  SS
SSSUT/5        [	        [
        SSS9UU 4S j5       nU" 5         g)zBTests that content type header is treated with case insensitivity.r  r  r  zcontent-Type:text/plainr   rf  r   r  c                  t   > TR                  SST/SS9n TR                  U S5        TR                  U S5        g )Nr  r  Tr  r   z	image/gifr   rY  assertNotRegexr  s    r   r  ATestCp.test_content_type_header_case_insensitive.<locals>._Check1y  s=    ~~tT62$~Gf
v<=
&,/r   zCONTENT-TYPE:image/gifzcontent-type:image/gifc                  t   > TR                  SST/SS9n TR                  U S5        TR                  U S5        g )Nr  r  Tr  r  zimage/gif,\s*image/gifr  r  s    r   r  ATestCp.test_content_type_header_case_insensitive.<locals>._Check2  s>    ~~tT62$~Gf
v;<
&";<r   N)rs  r   rG  r   rU   rm  )r   r   r   r  r  r  s   `    @r   )test_content_type_header_case_insensitive0TestCp.test_content_type_header_case_insensitiveo  s     ""$J*e$F
+ENND3T5&IJ >30 40
 INN&.Fv  >3= 4=
 Ir   c           	         U R                  5       n[        US5      nU R                  S5      nU R                  SSSSU R                  -  SX2/5        U R                  SSU/S	S
9nU R                  US5        U R                  US5        [        US5      nU R                  SX%/5        U R                  SSU/S	S
9nU R                  US5        U R                  US5        g)zETests that non-content-type headers are applied successfully on copy.r  r  r  Cache-Control:public,max-age=12x-%s-meta-1:abcdr   r  r  Tr  z%Cache-Control\s*:\s*public,max-age=12zMetadata:\s*1:\s*abcdr  N)rs  r   rG  r   provider_custom_metarY  )r   r   dst_urir   r  dst_uri2s         r   test_other_headersTestCp.test_other_headers  s     ""$J:u%G
+ENN/T666e 
 ^^T41^FFVEFV56J&HNND',-^^T42$^GFVEFV56r   c                 B   S[         R                  S'   U R                  5       n[        US5      nU R	                  S5      nU R                  SSX2/SS9nU R                  (       a  S	nOS
nU R                  XE5        U R                  SSSU/SS9nU R                  XE5        g)zJTest that x-goog-request-header can be set using the environment variable.b/this_is_env_reasonCLOUDSDK_CORE_REQUEST_REASONr  r  -DDr   TrV  z1b'X-Goog-Request-Reason': b'b/this_is_env_reason'z/'x-goog-request-reason': 'b/this_is_env_reason'r  r  N)r   environrs  r   rG  r   r   rY  )r   r   r*  r   r   reason_regexs         r   test_request_reason_header!TestCp.test_request_reason_header  s     2HBJJ-.""$J:u%G
+E^^UD%9^NFIlGlV*^^UD$8^MFV*r   z*XML APIs use a different debug log format.c                 \   S[         R                  S'   U R                  5       n[        US5      nU R	                  S5      nSn[        U/5         U R                  SSX2/SS	9nS
S
S
5        U R                  (       a  SnOSnU R                  WSU-   S-   U-   5        g
! , (       d  f       N@= f)FTest that x-goog-request-header works when cp sends multiple requests.r/  r0  r  r  r
  resumable_threshold0r1  r   TrV  Nz0X-Goog-Request-Reason\': b\'b/this_is_env_reasonz/x-goog-request-reason\': \'b/this_is_env_reasonz
GET[\s\S]*z[\s\S]*POST[\s\S]*)	r   r2  rs  r   rG  r4   r   r   rY  r   r   r*  r   boto_config_for_testr   r3  s          r   :test_request_reason_header_persists_multiple_requests_jsonATestCp.test_request_reason_header_persists_multiple_requests_json  s     2HBJJ-.""$J:u%G
+EA	34	5~~udE;4~Pf 
6 HlGl$'<<|KM 
6	5s   B
B+z+JSON API uses a different debug log format.c                 .   S[         R                  S'   U R                  5       n[        US5      nU R	                  S5      nSn[        U/5         U R                  SSX2/SS	9nS
S
S
5        SnU R                  WUS-   U-   5        g
! , (       d  f       N)= f)r7  r/  r0  r  r  r8  r  r   TrV  NzSFinal headers: \{[\s\S]*\'x-goog-request-reason\': \'b/this_is_env_reason\'[\s\S]*}z[\s\S]*)r   r2  rs  r   rG  r4   r   rY  r;  s          r   9test_request_reason_header_persists_multiple_requests_xml@TestCp.test_request_reason_header_persists_multiple_requests_xml  s     2HBJJ-.""$J:u%G
+EA	34	5~~tT5:$~Of 
6	E 
 	V\J6EF 
6	5s   B
Bc                    U R                  5       nU R                  USS9nU R                  USS9n[        U5      nU R                  S[	        U5      [	        U5      /5        U R                  XR                  5      nU R                  XR                  5       5      n[        U5      nU R                  US5        [        U5      nU R                  5       nU R                  SUR                  U/5        [        US5       nU R                  UR                  5       S5        SSS5        U R                  SUR                  < S	U< 3U/5        [        US5       nU R                  UR                  5       S5        SSS5        U R                  SUR                  < S	U< 3U/5        [        US5       nU R                  UR                  5       S5        SSS5        U R                  SUR                  < S	U< 3U/5        [        US5       nU R                  UR                  5       S5        SSS5        U R                  SUR                  < S	U< 3UR                  /5        U R                  SUR                  U/5        [        US5       nU R                  UR                  5       S5        SSS5        U R                  SXsR                  /S
SS9n	U R                   (       a  U R#                  SU	5        gU R#                  SU	5        g! , (       d  f       GN= f! , (       d  f       GN= f! , (       d  f       GNc= f! , (       d  f       GN= f! , (       d  f       N= f)zTests copy with versioning.   data2rq     data1r   data3r   s   data3N#Tr   r   r   zGdestination argument of the cp command cannot be a version-specific URLz'cannot be the destination for gsutil cp)CreateVersionedBucketr   urigenr   r   r  r   StorageUriCloneReplaceKeyget_keyStorageUriSetContentsFromStringr   versionless_urir   r   r   urir   r   )
r   r   k1_urik2_urig1g2g3r   r   r   s
             r   test_versioningTestCp.test_versioning  s    ++-J*xHF*xHF	BNND$v,V56,,Z9K9KLF++J8HIF	B((9	B!ENND&00%89	eT	a
qvvx* 
 	NNDV%;%;R@%HI	eT	a
qvvx* 
NNDV%;%;R@%HI	eT	a
qvvx* 
NNDV%;%;R@%HI	eT	a
qvvx* 
 	NN	**B	/1G1G	IJ 	NND&00%89	eT	a
qvvx* 
 ^^T5**5*.,-  /F 
mm'(.0 mm=vF? 
	
 
	 
	 
	 
	s<   >!L!L2!L/!M!M
L
L,/
L>
M
M!c                 D   ^  [        [        SSS9U 4S j5       nU" 5         g)z@Tests that copy all-versions errors when parallelism is enabled.rf  r   r  c            
         > TR                  SSS[        TR                  S5      [        TR                  S5      /SSS9n TR                  (       a  TR	                  S	U 5        g TR	                  S
U 5        g )N-mr   -Ar  r  r   Tr   z-sequential instead of parallel task executionz.-m option is not supported with the cp -A flag)r   r   rl  r   r   )r   r   s    r   rj  5TestCp.test_versioning_no_parallelism.<locals>._Check  ss    ~~
d
t++U
3
t++U
3
 /0,0  2f 
	!	!EvNFOr   N)rU   rm  )r   rj  s   ` r   test_versioning_no_parallelism%TestCp.test_versioning_no_parallelism  s)     >3P 4P Hr   z6S3 lists versioned objects in reverse timestamp order.c           	      P  ^ ^^^ T R                  5       mT R                  5       mT R                  5       mT R                  TSSS9nT R                  TSS[        U5      S9  T R                  TSSS9  T R                  TS	SS9  T R                  TS	SS9  T R	                  S
SS[        TS5      [        T5      /5        [        [        SSS9UUU 4S j5       nU" 5         T R	                  S
S[        TS5      [        T5      /5        [        [        SSS9UUU 4S j5       nU" 5         g)z#Tests cp -R with versioned buckets.r+  s   data0r  s   longer_data1r   r   r   gs_idempotent_generationrg   T)	versionedr   r   -RrY  *rf  r   r  c                    > T	R                  SS[        T5      /SS9R                  S5      n T	R                  SS[        T5      /SS9R                  S5      nT	R                  [	        U 5      S5        T	R                  [	        U5      S5        U S   R                  5       u  p#pCT	R                  U[        [	        S5      5      5        T	R                  [        U5      R                  S	5        US   R                  5       u  pSpcT	R                  U[        [	        S5      5      5        T	R                  [        U5      R                  S	5        U S
   R                  5       u  p#pCT	R                  U[        [	        S5      5      5        T	R                  [        U5      R                  S	5        US
   R                  5       u  pSpcT	R                  U[        [	        S5      5      5        T	R                  [        U5      R                  S	5        g)#Validates the results of the cp -R.r  -laTr  
   r   data0r+  r   longer_data1Nr   r   r  r   lenr   r
   r   )
r   r   size1_uri_str1size2uri_str2rw  rx  r   s
          r   r  ?TestCp.test_recursive_copying_versioned_bucket.<locals>._Check2G  s    ud;.? @.2   449E$K ud;.? @.2   449E$K  s8}a(
s8}a( 'qk//1e
uc#g,/0
{8,88#>&qk//1e
uc#g,/0
{8,88#> 'qk//1e
uc#n"567
{8,88#>&qk//1e
uc#n"567
{8,88#>r   c                    > TR                  SS[        T5      /SS9R                  S5      n TR                  SS[        T5      /SS9R                  S5      nTR                  [	        U 5      S5        TR                  [	        U5      S5        US   R                  5       u  p#pCTR                  U[        [	        S	5      5      5        TR                  [        U5      R                  S
5        g)rd  r  re  Tr  rf  rg  rf  r   ri  r+  Nrj  )r   r   rl  rm  rn  rw  bucket3_urir   s        r   r  ?TestCp.test_recursive_copying_versioned_bucket.<locals>._Check3i  s     ud;.? @.2   449E$K ud;.? @.2   449E$K  s8}a(
s8}a( 'qk//1e
uc#n"567
{8,88#>r   N)rH  r   rI  AssertNObjectsInBucketr   r   rU   rm  )r   v1_urir  r  rw  rx  rs  s   `   @@@r   'test_recursive_copying_versioned_bucket.TestCp.test_recursive_copying_versioned_bucket+  sJ    ,,.K,,.K,,.K ++.(0  2F 	"%./5f~  ?
 	Q$?Q$?Q$? 	NN	tT	k3		k		 >3? 4?4 I 	NND$[# 6[8IJK >3? 4?  Ir   z#Preconditions not supported for S3.c           	      .   U R                  5       nU R                  SS9nSnU R                  SUSU[        U5      /5        U R                  SUSU[        U5      /SSS9nU R                  (       a  U R                  S	U5        gU R                  S
U5        g)z?Tests that cp handles an object-not-exists precondition header.rD  rS  zx-goog-if-generation-match:0r  r   Tr   rG  zMHTTPError 412: At least one of the pre-conditions you specified did not hold.PreconditionExceptionNrs  r   r   r   r   r   )r   r   fpath1gen_match_headerr   s        r   test_cp_generation_zero_match$TestCp.test_cp_generation_zero_match|  s     ""$J  ( 3F5 	NND*D&$z:JKL ^^	v	j			  F
 
mm"$ mm+V4r   c           
      ^   U R                  5       nU R                  USS9nUR                  nU R                  5       nU R	                  USS9nSU-  nU R                  SUSU[        U5      /5        U R                  SUSU[        U5      /SS	S
9nU R                  (       a  U R                  SU5        OU R                  SU5        U R                  SUSSU[        U5      /SS	S
9nU R                  (       a  U R                  SU5        gU R                  SU5        U R                  SU5        g)z?Tests that cp -v option handles the if-generation-match header.rD  rq  rC  r   r   zx-goog-if-generation-match:%sr  r   Tr   rG  zpre-conditionrz  rU  z:Cannot specify both generation precondition and no-clobberArgumentExceptionzASpecifying x-goog-if-generation-match is not supported with cp -nN)	rH  r   
generationr  r   r   r   r   r   )r   r   rO  rQ  r   r|  r}  r   s           r   test_cp_v_generation_match!TestCp.test_cp_v_generation_match  sJ    ++-J*xHF			B!F   BF6;NND*D&$v,GH ^^	v	f		  F 
mmOV,
mm+V4 ^^	tV	f		  F 
mm
FP mm'0
mm r   c                    U R                  5       nU R                  USS9nU R                  5       nU R                  USS9nU R	                  SSU[        U5      /5        U R	                  SSU[        U5      /SS9nU R                  S	U5        g
)z4Tests that cp -nv works when skipping existing file.rD  rq  rC  r  r   z-nvTrV  zSkipping existingN)rH  r   r  r   r   r   r   )r   r   rO  r   r|  r   s         r   
test_cp_nvTestCp.test_cp_nv  s     ++-J*xHF!F   BF 	NND%f67 ^^	ufd6l+4  AFMM%v.r   c                    U R                  5       nU R                  USS9nU R                  USS9nU R                  5       nU R                  USS9nU R	                  SXSR
                  5        [        nSS[        U5      4n[        U/5         [        R                  " U5      nU R                  5       nU R                  XHS9nU R	                  SXSR
                  5        SSS5        U R	                  SS	UR
                  5        U R                  5       nU R                  US
9n[        U5      n	U R                  SS[        U5      [        U	5      /SS9n
U R                  SU	R
                  -  U
5        U R	                  SUR
                  UR
                  5        U R	                  SUR
                  UR
                  5        g! , (       d  f       N= f)zD"Tests that cp -v returns the created object's version-specific URI.rD  rq  rC  r  -vr
  r9  Nr  r   r   TrV  zCreated: %s
z-Dv)rH  r   r  r   _run_cp_minus_v_testrN  r[   r   r4   r   urandomr
   r   r   r   )r   r   rO  rP  r   r|  size_thresholdr<  file_as_stringr*  r   s              r   test_cp_v_optionTestCp.test_cp_v_option  s    ++-J*xHF*xHF !F   BFdFJJ7 N$&;/1	34	5zz.1n!!#f""&"Jf
fjj9	 
6 	dC4 !F   /F&!G^^	tT&\4=1  GF 	MM/GKK/8 	eVZZ< 	dFJJ

;1 
6	5s   AG
Gc                    ^ ^^ T R                  SXT/SS9n[        R                  " SU5      nT R                  U5        UR	                  S5      m[        [        SSS9UUU 4S j5       nU" 5         g	)
z6Runs cp -v with the options and validates the results.r   TrV  zCreated: (.*)\nr   rf  r  c                     > TR                  SST/SS9n U R                  S5      nTR                  [        U5      S5        TR	                  TUS   5        g )Nr  -aTr  rf  rg   )r   r  assertGreaterrk  r   )r  r  created_uridst_strr   s     r   r  ,TestCp._run_cp_minus_v_test.<locals>._Check1  sT    ~~tT734~Hfll4 e UQ'
{E"I.r   N)r   r   r   assertIsNotNonegrouprU   rm  )r   optsrc_strr  r   matchr  r  s   `  `   @r   r  TestCp._run_cp_minus_v_test   sl    ^^T39^NFII(&1E++a.K >3/ 4/ Ir   c                 *  ^ ^^^ T R                  5       nT R                  USS9mT R                  USS9mT R                  5       mT R                  SS[	        T5      /SR                  TT45      S9  [        [        SS	S
9UUUU 4S j5       nU" 5         g)zTests cp with the -I option.rD  r  rC  r   z-Irf  )r  rf  r   r  c                    > TR                  S[        T5      /SS9n TR                  [        R                  R                  T5      U 5        TR                  [        R                  R                  T5      U 5        TR                  U S5        g )Nr  Tr  rg   )r   r   r   r   r   basenameassertNumLines)r  r   r|  fpath2r   s    r   r  'TestCp.test_stdin_args.<locals>._Check1  si    ~~tT*%56d~Kf
mmBGG$$V,f5
mmBGG$$V,f5
&!$r   N)r  r   rs  r   r   r   rU   rm  )r   r   r  r   r|  r  s   `  @@@r   test_stdin_argsTestCp.test_stdin_args  s     !F   BF   BF""$JNND$Z 01FF#34  6 >3% 4% Ir   c                     U R                  SS9nU R                  SS9nU R                  USS9nU R                  S[        U5      [        U5      /5        g )Nstandardstorage_classdurable_reduced_availabilityr   rq  r   rs  r   r   r   )r   rw  rx  r^  s       r   !test_cross_storage_class_cloud_cp(TestCp.test_cross_storage_class_cloud_cp'  s^    ##*#=K##4 $ 6K;HGNND$w-k):;<r   z(Test requires both S3 and GS credentialsc                    U R                  SS9nU R                  SS9nU R                  USS9nU R                  USS9nU R                  S[        U5      [        U5      /5        U R                  S[        U5      [        U5      /5        g )Ns3providergsr   rq  rT  r   r  r   	s3_bucket	gs_buckets3_keygs_keys        r   test_cross_provider_cpTestCp.test_cross_provider_cp/  s    !!4!0I!!4!0I)fEF)fEFNND$v,Y89NND$v,Y89r   zHThis test performs a large copy but remains here for debugging purposes.c           	         U R                  SS9nU R                  SS9nU R                  USS-  S9nU R                  USS-  S9nU R                  S[        U5      [        U5      /5        U R                  S[        U5      [        U5      /5        [	        S	S
[        [        5      4S	S[        [        S-  5      4/5         U R                  S[        U5      [        U5      /5        S S S 5        g ! , (       d  f       g = f)Nr  r  r  s   ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffi   rq  s   bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbr   r
  r9  json_resumable_chunk_size   )rs  r   r   r   r4   r   r[   r  s        r   test_cross_provider_large_cp#TestCp.test_cross_provider_large_cp8  s     !!4!0I!!4!0I)(3d(:  <F)(3d(:  <FNND$v,Y89NND$v,Y89	*?WN (*E #GcM 2 45 
6 nndDL$y/:;	
6 
6 
6s   'C11
C?c                     U R                  SS9nU R                  SSS9n[        [        S-  5      nU R                  USU-  SS9nU R	                  S	S
S	SS[        U5      [        U5      /5        g)zEEnsure daisy_chain works for an object that is downloaded in 2 parts.r  r  r  T)r  r  g?   b)r   r   r  z-ozs3:use-sigv4=Truezs3:host=s3.amazonaws.comr   N)rs  r   r   r   r   r   )r   r  r  	num_bytesr  s        r   test_gs_to_s3_multipart_cp!TestCp.test_gs_to_s3_multipart_cpJ  s     !!4!0I!!4!FI0367I)(,y(8/3  5F 	NN!4)CTVY r   zXThis test is slow due to creating many objects, but remains here for debugging purposes.c           	         U R                  5       nU R                  5       nSn[        U5       HQ  nSU-  S-
  nSU-  nSU-  S-   nU R                  USU-  S9  U R                  USU-  S9  U R                  USU-  S9  MS     U R                  XS-  5        U R	                  S	S
S[        US5      [        U5      /5        U R                  X#S-  5        g)z6Ensure daisy chain cp works with a wide of file sizes.   rg   r      arq  r     crf  rX  r   r  z**N)rs  r_   r   ru  r   r   )r   r   rx  exponent_capione_byte_smallernormalone_byte_largers           r   test_daisy_chain_cp_file_sizes%TestCp.test_daisy_chain_cp_file_sizesY  s     ""$J##%KL< A!tf1qo
:?O8OP
:vF
:8NO ! 	
1,<=NN	tT	j$		k		
 	A-=>r   c                   ^ ^^^ T R                  SS9nT R                  SS9mT R                  USS9mT R                  SSSSS	SS
T R                  -  [	        T5      /5        T R                  SSS[	        T5      /5        T R                  SS[	        T5      /SS9mT R                  SS[	        T5      [	        T5      /SS9nT R                  SU5        [        [        SSS9UUUU 4S j5       nU" 5         g)zTests cp with the -D option.r  r  r  r   rq  setmetar  r'  r  r(  aclr   public-readr   Tr  r   z-DpnrV  zCopy-in-the-cloud disallowedrf  r   r  c                    > [        TTR                  5      n TR                  SSU /SS9nTR                  US5        TR                  US5        TR                  US5        TR                  SS	U /SS9nTR	                  TU5        g )
Nr  r  Tr  z"Cache-Control:\s+public,max-age=12r  zMetadata:\s+1:\s+abcdr  r   )r   r   r   rY  r   )rN  r  new_acl_jsonacl_jsonrx  r^  r   s      r   rj  *TestCp.test_daisy_chain_cp.<locals>._Check  s    g112c~~tT3/t~Df
vDE
v;<
v78^^UE3$7t^Ll
x.r   N)rs  r   r   r)  r   assertNotInrU   rm  )r   rw  r   rj  r  rx  r^  s   `   @@@r   test_daisy_chain_cpTestCp.test_daisy_chain_cpp  s   ##*#=K##4 $ 6K;HG 	NN4:D $T666W	  	NNE5-g?@~~ueT'];4~PH
 ^^	vtG}	k		+/  1F 	3V<
>3/ 4/ Hr   zpgs_port is defined in config which can cause problems when uploading and downloading to the same local host portc                    U R                  5       nU R                  5       nU R                  USU R                  -  S9nSS[        [        5      4nU R                  [        R                  " [        SS5      5      S9n[        U/5         U R                  SS	US
[        U5      [        U5      /SSS9nU R                  UR                  S5      S5        SSS5        g! , (       d  f       g= f)z:Tests cp with the -D option when the download thread dies.r  rq  r
  r9  Fr  rS  r   --testcallbackfiler  r   Tr   z8ResumableDownloadException: Artifically halting downloadrf  N)rs  r   	halt_sizer   r[   r   pickledumpsr(   r4   r   r   r   ru  )r   rw  rx  r^  r<  test_callback_filer   s          r   $test_daisy_chain_cp_download_failure+TestCp.test_daisy_chain_cp_download_failure  s    
 ##%K##%K;)-)>   @G$&;S\J,,8BC - E	34	5~~
$&8$
w-
{

 /0,0  2f 
,,HJKLN 
6	5	5s   	AC
C+c           	          U R                  5       nU R                  SSS[        US5      /SSSS9nU R                  (       a  U R	                  S	U5        gU R	                  S
U5        g)zETests error when compression flag is requested on a streaming source.r   -Zr  r  Tr   zstreaming data)r   r   r  zGGzip content encoding is not currently supported for streaming uploads.z@gzip compression is not currently supported on streaming uploadsNr  r  s      r   test_streaming_gzip_upload!TestCp.test_streaming_gzip_upload  sr    ""$J^^	tS$z512	   F
 
mm mm
L
r   c           
         U R                  SS9nU R                  5       n[        SS/5         U R                  SSSU[	        U5      /SS	9nU R                  S
U5        SSS5        [        SS/5         U R                  SSSU[	        U5      /SS	9nU R                  SU5        SSS5        g! , (       d  f       NU= f! , (       d  f       g= f)z?Tests that the seek-ahead iterator estimates total upload work.rf  
test_filesr
  task_estimation_threshold1r
  task_estimation_forceTruerX  r   r  TrV  ;Estimated work for this command: objects: 3, total size: 18Nr
  r  r:  Estimated work)r  rs  r4   r   r   r   r  r   r   r   r   s       r   test_seek_ahead_upload_cp TestCp.test_seek_ahead_upload_cp  s    1-F""$J	KJL 
M~~tVT*%5
6d  Lf
mm
GQ	
M 
KJL 
M~~tVT*%5
6d  Lf
'0	
M 
M
M 
M
M 
Ms   1B-31B>-
B;>
Cc           	         U R                  5       nU R                  SS9nU R                  US5        [        SS/5         U R	                  SSS[        U5      U/SS	9nU R                  S
U5        S S S 5        [        SS/5         U R	                  SSS[        U5      U/SS	9nU R                  SU5        S S S 5        g ! , (       d  f       NU= f! , (       d  f       g = f)Nrf  test_objectsr  r  rX  r   r  TrV  r  r  r  )r  rs  ru  r4   r   r   r   r  r  s       r   test_seek_ahead_download_cp"TestCp.test_seek_ahead_download_cp  s    !F"""2J
A.	KJL 
M~~tT*-v
6d  Lf
mm
GQ	
M 
KJL 
M~~tT*-v
6d  Lf
'0	
M 
M
M 
M
M 
Ms   1B?1C?
C
Cc           	      t  ^ ^^^ T R                  5       nT R                  5       mT R                  USS9mT R                  SSS[        T5      [        T5      /5        T R                  SSS[        T5      /5        T R                  SS[        T5      /S	S
9m[	        [
        SSS9UUUU 4S j5       nU" 5         g)z Tests copying with a canned ACL.r   rq  r   r  r  r  r   r   Tr  rf  r   r  c                  |   > [        TTR                  5      n TR                  SSU /SS9nTR                  TU5        g )Nr  r   Tr  )r   r   r   r   )rN  r  rx  r^  public_read_aclr   s     r   rj  )TestCp.test_canned_acl_cp.<locals>._Check  s>    g112c^^UE3$7t^Ll
5r   N)rs  r   r   r   rU   rm  )r   rw  rj  rx  r^  r  s   `  @@@r   test_canned_acl_cpTestCp.test_canned_acl_cp  s    ##%K##%K;HGNN	t]	g	k		 	NNE5-g?@nneUDM%B37 % 9O >36 46
 Hr   c           
         U R                  5       nU R                  USS9nU R                  SSS[        U5      /5        U R                  SS[        U5      /SS9nS	nU R	                  USS
9nU R                  SSSU[        U5      /5        U R                  SS[        X5      /SS9nU R                  X65        [        nSS[        U5      4n[        U/5         Sn	[        R                  " U5      n
U R	                  U	U
S
9nU R                  SSSU[        U5      /5        U R                  SS[        X5      /SS9nU R                  X<5        SSS5        g! , (       d  f       g= f)z)Tests uploading a file with a canned ACL.r   rq  r  r   r  r   Tr  r  rC  r   r  r
  r9  resumable_barN)rs  r   r   r   r   r   r[   r   r4   r   r  )r   rw  r^  r  rD  r   r  resumable_sizer<  resumable_file_nameresumable_contentsresumable_fpathnew_resumable_acl_jsons                r   test_canned_acl_uploadTestCp.test_canned_acl_upload  s    ##%K;HGNNE5-g?@nneUDM%B37 % 9O I)fEENND$ud;6GHI>>	tK34D " JL_3N$&;/1	34	5+::n5++6I5G , Io
nn}o  $~~%k?
@  .   ? 
6	5	5s   A5E
E%c                     U R                  5       nSnU R                  XS9nU R                  S[        U5      S/SS9nU R	                  X$R                  S5      5        g )Nr   rq  r   r  Tr  r  )rs  r   r   r   r   r   )r   r   r   r^  r  s        r   test_cp_key_to_local_stream"TestCp.test_cp_key_to_local_stream  sZ    ""$JH:IG^^T4=#6d^KFMM(MM'23r   c                     SnU R                  US9nU R                  SUS/SS9nU R                  XR                  [        5      5        g )Ns   contentrS  r   r  Tr  )r   r   r   r   rD   )r   r   r   r  s       r   "test_cp_local_file_to_local_stream)TestCp.test_cp_local_file_to_local_stream%  sH    H2E^^T5#.d^CFMM(MM$/0r   c                   ^ ^^ T R                  5       mT R                  5       n[        R                  R	                  US5      m[        TS5       n S S S 5        T R                  ST[        T5      /5        [        [        SSS9UUU 4S j5       nU" 5         [        R                  R	                  US5      nT R                  S[        TS5      U/5        T R                  [        R                  " U5      5        g ! , (       d  f       N= f)	N	zero_bytewr   rf  r   r  c                     > TR                  S[        T5      /SS9n TR                  [        R                  R                  T5      U 5        g )Nr  Tr  )r   r   r   r   r   r  )r  dst_bucket_urir   r   s    r   r  .TestCp.test_cp_zero_byte_file.<locals>._Check14  s<    ~~tT.%9:$~Of
mmBGG$$U+V4r   zero_byte_download)rs  r  r   r   r   r   r   r   rU   rm  r   r   )r   src_dirunused_out_filer  download_pathr  r   s   `    @@r   test_cp_zero_byte_fileTestCp.test_cp_zero_byte_file+  s    &&(N  "GGGLL+.E	eS	_
 
NND%n!567
>35 45 IGGLL*>?MNND$~{;]KLOOBGGM*+ 
	s   C<<
D
c                    ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  [        [        SSS9UU U4S	 j5       nU" 5         g
)zTests recursively copying from bucket to bucket.

This should produce identically named objects (and not, in particular,
destination objects named by the version-specific URI from source objects).
obj0   abcr  r      defrf  r   r  c                    > TR                  SS[        T5      [        T5      /5        TR                  SSTR                  /SS9n TR                  T< TR                  < S3U 5        TR                  T< TR                  < S3U 5        g Nr   ra  r  Tr  z/obj0
z/obj1
r   r   rN  r   r   r  r  r   src_bucket_uris    r   _CopyAndCheck8TestCp.test_copy_bucket_to_bucket.<locals>._CopyAndCheckO      
nndD$~"6^8LMN~~tT>+=+=>,0  2f
mm)>+E+E
FP
mm)>+E+E
FPr   N)rH  r   rU   rm  )r   r&  r  r%  s   ` @@r   test_copy_bucket_to_bucket!TestCp.test_copy_bucket_to_bucket?  s     //1N//1N"(%  ' 	"(%  '
 >3P 4P Or   c                    ^ ^ T R                  5       mT R                  TSSS9  T R                  TSSS9  [        [        SSS9UU 4S j5       nU" 5         g	)
BTests copying from bucket to same bucket preserves file structure.dir1/file.txtr}  r  dir2/file.txtrf  r   r  c                    > TR                  SS[        T5      S-   [        T5      S-   /5        TR                  SSTR                  /SS9n TR                  [        T5      S-   U 5        TR                  [        T5      S	-   U 5        g )
Nr   ra  /*/dstr  Tr  /dst/dir1/file.txt/dst/dir2/file.txtr   r   rN  r   r  r   r   s    r   r&  VTestCp.test_copy_duplicate_nested_object_names_to_new_cloud_dir.<locals>._CopyAndCheckf  s    
nn
d"
f$&' ~~tT:>>:$~Of
mmD$';;VD
mmD$';;VDr   Nrs  r   rU   rm  r   r&  r   s   ` @r   8test_copy_duplicate_nested_object_names_to_new_cloud_dir?TestCp.test_copy_duplicate_nested_object_names_to_new_cloud_dir[  so    ""$J"1&  ( 	"1&  (
 >3E 4E Or   c                    ^ ^ T R                  5       mT R                  TSSS9  T R                  TSSS9  T R                  TSSS9  [        [        SSS9UU 4S	 j5       nU" 5         g
)r,  r-  r}  r  r.  zdst/existing_file.txtrf  r   r  c                  N  > TR                  SS[        T5      S-   [        T5      S-   /5        TR                  SSTR                  /SS9n TR                  [        T5      S-   U 5        TR                  [        T5      S	-   U 5        TR                  [        T5      S
-   U 5        g )Nr   ra  r0  r1  r  Tr  r2  r3  z/dst/existing_file.txtr4  r5  s    r   r&  [TestCp.test_copy_duplicate_nested_object_names_to_existing_cloud_dir.<locals>._CopyAndCheck  s    
nn
d"
f$&' ~~tT:>>:$~Of
mmD$';;VD
mmD$';;VD
mmD$'??Hr   Nr7  r8  s   ` @r   =test_copy_duplicate_nested_object_names_to_existing_cloud_dirDTestCp.test_copy_duplicate_nested_object_names_to_existing_cloud_dirr  s    ""$J"1&  ( 	"1&  ( 	"9&  (
 >3I 4I Or   z3Only s3 V4 signatures error on location mismatches.c                   ^ ^^ SnSnSU-  nSU-  nSn[        SSU4/5         T R                  US9mT R                  TSS	S
9  T R                  TSSS
9  S S S 5        [        SSU4/5         T R                  US9mS S S 5        [        [        SSS9UU U4S j5       n[        SSU4/5         U" 5         S S S 5        g ! , (       d  f       No= f! , (       d  f       NY= f! , (       d  f       g = f)Nz	ap-east-1	us-east-2zs3.%s.amazonaws.comzs3.eu-west-1.amazonaws.comr  host)locationr  r  r  r   r   rf  r   r  c                    > TR                  SS[        T5      [        T5      /5        TR                  SSTR                  /SS9n TR                  T< TR                  < S3U 5        TR                  T< TR                  < S3U 5        g r"  r#  r$  s    r   r&  OTestCp.test_copy_bucket_to_bucket_with_location_redirect.<locals>._CopyAndCheck  r(  r   )r4   rs  r   rU   rm  )	r   src_bucket_regiondest_bucket_regionsrc_bucket_hostdest_bucket_hostclient_hostr&  r  r%  s	   `      @@r   1test_copy_bucket_to_bucket_with_location_redirect8TestCp.test_copy_bucket_to_bucket_with_location_redirect  s    $$+.??O,/AA.K	fo>?	@((2C(Dn
>$*!'  ) >$*!'  ) 
A 
f.>?@	A((2D(En 
B >3P 4P 
fk:;	<o 
=	</ 
A	@ 
B	A 
=	<s#   2B>,C-C >
C
C 
C.c                    ^ ^^ T R                  5       mT R                  5       mT R                  TSSS9  T R                  TSSS9  [        [        SSS9UU U4S	 j5       nU" 5         g
)zTests recursively copying from bucket to a directory.

This should produce identically named objects (and not, in particular,
destination objects named by the version- specific URI from source objects).
r  r  r  r   r   rf  r   r  c                  8  > TR                  SS[        T5      T/5        / n [        R                  " T5       H=  u  pnU H1  nU R	                  [        R
                  R                  X5      5        M3     M?     [        U 5      n TR                  [        U 5      S5        TR                  [        R
                  R                  TTR                  S5      U S   5        TR                  [        R
                  R                  TTR                  S5      U S   5        g	z8Copies the bucket recursively and validates the results.r   ra  rg   r  r   r   r   Nr   r   r   walkr   r   r   sortedr   rk  r   dir_listdirnamerm  	filenamesfilenamedst_dirr   r%  s        r   r&  5TestCp.test_copy_bucket_to_dir.<locals>._CopyAndCheck       nndD$~"6@Ah#%777#3
'i!H
//"'',,w9
: " $4 !h
s8}a(

'',,w : :F
C
1+ 
'',,w : :F
C
1+r   Nrs  r  r   rU   rm  r   r&  rX  r%  s   ` @@r   test_copy_bucket_to_dirTestCp.test_copy_bucket_to_dir  s}     &&(N  "G"(%  ' 	"(%  '
 >3 4  Or   c                    ^ ^^ T R                  SSS9mT R                  5       mT R                  TSSS9  T R                  TSSS9  [        [        S	S
S9UU U4S j5       nU" 5         g)zTests copying object from s3 to local dir with v4 signature.

Regions like us-east2 accept only V4 signature, hence we will create
the bucket in us-east2 region to enforce testing with V4 signature.
r  rA  )r  rC  r  r  r  r   r   rf  r   r  c                  8  > TR                  SS[        T5      T/5        / n [        R                  " T5       H=  u  pnU H1  nU R	                  [        R
                  R                  X5      5        M3     M?     [        U 5      n TR                  [        U 5      S5        TR                  [        R
                  R                  TTR                  S5      U S   5        TR                  [        R
                  R                  TTR                  S5      U S   5        grO  rP  rS  s        r   r&  ;TestCp.test_copy_object_to_dir_s3_v4.<locals>._CopyAndCheck  rZ  r   Nr[  r\  s   ` @@r   test_copy_object_to_dir_s3_v4$TestCp.test_copy_object_to_dir_s3_v4  s     &&{&KN  "G"(%  ' 	"(%  '
 >3 4  Or   z]The boto lib used for S3 does not handle objects starting with slashes if we use V4 signaturec                 "   U R                  5       nU R                  5       nU R                  USSS9  U R                  USSS9  U R                  US5      nU R	                  US5        U R                  US5        U R                  S	S
[        U5      U/5        / n[        R                  " U5       H=  u  pVnU H1  nUR                  [        R                  R                  XX5      5        M3     M?     [        U5      nU R                  [        U5      S5        U R                  [        R                  R                  X!R                   S5      US   5        U R                  [        R                  R                  X!R                   S5      US   5        g)9Tests that we correctly handle leftover dir placeholders.r  r  r  r   r   r   r   rf  r   ra  rg   r   r   Nrs  r  r   r  rL  ru  r   r   r   rQ  r   r   r   rR  r   rk  r   	r   r%  rX  r^  rT  rU  rm  rV  rW  s	            r   @test_recursive_download_with_leftover_slash_only_dir_placeholderGTestCp.test_recursive_download_with_leftover_slash_only_dir_placeholder  s^    &&(N  "G"(%  ' 	"(%  '
 --ncBG(("52NND$^ 4g>?H!#!1I(W78   "2 hHS]A&RWW\\'+E+EvNa["RWW\\'+E+EvNa["r   c                 "   U R                  5       nU R                  5       nU R                  USSS9  U R                  USSS9  U R                  US5      nU R	                  US5        U R                  US5        U R                  S	S
[        U5      U/5        / n[        R                  " U5       H=  u  pVnU H1  nUR                  [        R                  R                  XX5      5        M3     M?     [        U5      nU R                  [        U5      S5        U R                  [        R                  R                  X!R                   S5      US   5        U R                  [        R                  R                  X!R                   S5      US   5        g)re  r  r  r  r   r   zfoo/r   rf  r   ra  rg   r   r   Nrf  rg  s	            r   5test_recursive_download_with_leftover_dir_placeholder<TestCp.test_recursive_download_with_leftover_dir_placeholder  s\   &&(N  "G"(%  ' 	"(%  '
 --nfEG(("52NND$^ 4g>?H!#!1I(W78   "2 hHS]A&RWW\\'+E+EvNa["RWW\\'+E+EvNa["r   c                     U R                  5       nU R                  USS9nU R                  SS[        U5      [        U R	                  US5      5      /SS9nU R                  UR                  S5      S	5        g )
Nr   rq  z-qr   o2TrV  zCopying r   )rs  r   r   r   r  r   ru  )r   r   r^  r   s       r   test_copy_quietTestCp.test_copy_quiet4  s~    ""$J:GG^^dWT,,Z>?
 +/  0F 	V\\*-q1r   c                   ^ ^^ T R                  5       mT R                  SS9n[        US5       n[        R                  " [        U5      5      n [        R                  " U5      nUR                  S5      mSSS5        T R                  SU[        T5      /5        [        [        SSS	9UUU 4S
 j5       nU" 5         g! [         a    [        R                  " U5      n Nzf = f! , (       d  f       Ns= f)zTests that the uploaded object has the expected MD5.

Note that while this does perform a file to object upload, MD5's are
not supported for composite objects so we don't use the decorator in this
case.
rT  rS  r      
Nr   rf  r   r  c            	         > TR                  SS[        T5      /SS9n TR                  U S[        R                  " TR                  S5      5      -  5        g )Nr  r  Tr  zHash\s+\(md5\):\s+%sr  )r   r   rY  r   r[  decode)r  r   file_md5r   s    r   r  )TestCp.test_cp_md5_match.<locals>._Check1S  sS    ~~tT4
+;<,0  2f

)BIIhoog6N,OOQr   )rs  r   r   binascii	unhexlifyrJ   base64encodebytesAttributeErrorencodestringrstripr   r   rU   rm  )r   r   f_inmd5encoded_bytesr  r   ru  s   `     @@r   test_cp_md5_matchTestCp.test_cp_md5_match?  s     ""$J0E	eT	d7=>c1**3/ %%e,h 
 	NND%j!123 >3Q 4Q I  1++C01	 
	s/    C$B>&C$> C!C$ C!!C$$
C2z:Unicode handling on Windows requires mods to site-packagesc                     U R                  SR                  [        5      SR                  [        5      SR                  [        5      5      $ )Nu   foo-unicödeu   bar-unicödeu   manifest-unicöde)_ManifestUploadr   rD   r   s    r   test_cp_manifest_upload_unicode&TestCp.test_cp_manifest_upload_unicode\  sB      5 5d ; . 5 5d ; 3 : :4 @B Br   c                 (    U R                  SSS5      $ )z$Tests uploading with a mnifest file.r  r  manifest)r  r  s    r   test_cp_manifest_uploadTestCp.test_cp_manifest_uploadd  s     uj99r   c                 
   U R                  5       n[        XB5      nU R                  USS9nU R                  USS9n[        US5      R	                  5         U R                  SSXvU/5        [        US5       nUR                  5       n	SSS5        [        R                  (       a+  W	 V
s/ s H  n
[        R                  " U
[        5      PM      n	n
U R                  [        W	5      S	5        / S
QnU R                  XS   R                  5       R                  S5      5        U	S   R                  5       R                  S5      n[        [!        X5      5      nU R                  US   SU-   5        U R                  US   U5        Sn["        R"                  R%                  US   U5      n["        R"                  R%                  US   U5      nU R                  X:  S5        U R
                  [&        R(                  R
                  :X  a  U R                  US   S5        U R                  [+        US   5      S5        U R                  [+        US   5      S5        U R                  US   S5        g! , (       d  f       GN= fs  sn
f )z%Tests uploading with a manifest file.rT  rC  r   r  r   r  r  Nrg   
SourceDestinationStartEndMd5UploadIdSource SizeBytes TransferredResultDescriptionr   r  r   r  file://r  %Y-%m-%dT%H:%M:%S.%fZr  r  Tr  zrL0Y20zC+Fzt72VPzMSk2A==r  rf  r  r  OK)rs  r   r   r   closer   r  r   PY2	text_typerD   r   rk  r  r  dictzipdatetimestrptimetestcaseGsUtilIntegrationTestCaser   )r   rD  r   manifest_namer   r  r   logpathr   r  lineexpected_headersr  date_format
start_dateend_dates                   r   r  TestCp._ManifestUploadi  s/   ""$J**F)fEE!!MC!HG#NND$78	gs	qkkme 

ww5:;UTs}}T4(Ue;SZ# 	%Qx~~'7'='=c'BCAhnn$$S)G3'12GE 	
 *K""++GG,<kJJ  ))'%.+FHX*D1~~;;EEE
 wu~'ABS/0!4S!456:WX&-O 
	 <s   7I.)%J .
I=c                    U R                  SS9nU R                  SS9nU R                  SS9n[        US5      R                  5         U R	                  SSU[        U5      U/SS9  [        US	5       nUR                  5       nS
S
S
5        [        R                  (       a  / nW H  nUR                  S5      (       a  UR                  S5      n/ n	U Hd  n
U
R                  S5      (       a:  U	R                  [        R                  " U
5      R                  [        5      5        MS  U	R                  U
5        Mf     UR                  SR!                  U	5      5        M  UR                  U5        M     UnU R#                  [%        W5      S5        / SQnU R#                  XS   R'                  5       R                  S5      5        US   R'                  5       R                  S5      nU R#                  US   S
S SU R(                  -  5        U R#                  US   S
S S5        Sn[*        R*                  R-                  US   U5      n[*        R*                  R-                  US   U5      nU R#                  X:  S5        U R#                  [/        US   5      S5        U R1                  [/        US   5      S5        U R#                  US   S5        g
! , (       d  f       GNT= f)z'Tests downloading with a manifest file.r   rS  r   r  r   r  Tr  r  Nzb'r  rg   r  r   r   r  z%s://   r  r  rf     r   r  )r   r   r   r  r   r   r  r   PY3
startswithr  r   astliteral_evalrt  rD   r   r   rk  r  r   r  r  r   rt  )r   r^  r   r  r   r  decode_linesr  	some_strs
line_partssome_strr  r  r  r  r  s                   r   test_cp_manifest_download TestCp.test_cp_manifest_download  s    0G-E!!3!/G#NN	tWd7mU34  I	gs	qkkme 

wwl$??4  jjo)*#h""4(( 0 0 : A A$ GH)	 $
 

chhz2
3


d
#  eSZ# 	%Qx~~'7'='=c'BCAhnn$$S)GWQZ^Wt/D/D%DEWQZ^Y/)K""++GAJDJ  ))'!*kBHX*D1S_a( 	C
OQ/WQZ&I 
	s   2K
Kc                 ~   U R                  5       nS[        -  S-  nU R                  SUS9n[        S/5         U R	                  SU[        U5      /SS9  U R	                  S	[        U5      /SSS
9nU R                  UR                  S5      U5        S S S 5        [        SS[        [        S-  5      4/5         U R	                  SU[        U5      /SS9  U R	                  S	[        U5      /SSS
9nU R                  UR                  S5      U5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)N   xrg   u   АудиоархивrC  )r
  r9  r  r   TrV  cat)r  force_gsutilr  r
  r9  rf  )	r   rC   r   r4   r   r   r   r   r   )r   r^  file_contentsr   r  s        r   $test_copy_unicode_non_ascii_filename+TestCp.test_copy_unicode_non_ascii_filename  sK   !G 33a7M*@=YE	EF	G
nndE4=1nF~~ud7m4,0+/  1f v}}W-}= 
H 
*? #$<q$@ A C D 
E
nndE4=1nF~~ud7m4,0+/  1f v}}W-}=
E 
E 
H	G
E 
Es   AD:AD.
D+.
D<c           	      j   U R                  5       nSS-  nU R                  5       nU R                  SX2S9  U R                  SX2S9  U R                  SX2S9  U R                  SSS	[        R
                  R                  US
5      [        U5      /5        U R                  US5        [        US5      n[        US5      n[        US5      nU R                  SU/SS9nU R                  US5        U R                  SU/SS9nU R                  US5        U R                  SU/SS9nU R                  US5        U R                  5       nXEU4 HU  n	U R                  SU	[        U5      /5        [        US5       n
U R                  U
R                  5       U5        S S S 5        MW     g ! , (       d  f       Mi  = f)Nr  '  	test.htmlrD  r   r   test.jstest.txtr   -zjs, htmlztest.*rf  r   Tr  Content-Encoding:\s+gzipr   )rs  r  r   r   r   r   r   r   ru  rY  r   r   r   r   )r   r   r   r   uri1uri2uri3r  fpath4rN  r   s              r   test_gzip_upload_and_download$TestCp.test_gzip_upload_and_download  s   ""$Je|H!F+fP)FN*VO 	NNdJ
VX&Z 
 	
A.
K(D
I&D
J'D^^VTN$^?FV89^^VTN$^?FV89^^VTN$^?F ;<  "FD!
nndCf./8,  "s   5!F##
F2	z0No compressed transport encoding support for S3.z9No compressed transport encoding support for the XML API.c                    ^  U 4S jnU 4S jnU 4S jnSS-  4U 4S jjnU" 5       u  pVnU" Xe5      nU" Xx5        U" U5        g)zTest gzip encoded files upload correctly.

This checks that files are not tagged with a gzip content encoding and
that the contents of the files are uncompressed in GCS. This test uses the
-j flag to target specific extensions.
c            	         > TR                  5       n SS-  nTR                  5       n/ nS H#  nUR                  TR                  UUUS95        M%     XU4$ )au  Setup the bucket and local data to test with.

Returns:
  Triplet containing the following values:
    bucket_uri: String URI of cloud storage bucket to upload mock data
                to.
    tmpdir: String, path of a temporary directory to write mock data to.
    local_uris: Tuple of three strings; each is the file path to a file
                containing mock data.
r  r  )r  r  r  r  )rs  r  r   r   )r   r   r   
local_urisrW  r   s        r   _create_test_dataQTestCp.test_gzip_transport_encoded_upload_and_download.<locals>._create_test_data	  so     $$&jh!!#fj:(('-)1   3	4 ; *--r   c           
         > TR                   (       a  SnOSnTR                  SSSU[        R                  R	                  U S5      [        U5      /SS9nTR                  US	5        U$ )
a  Upload local test data.

Args:
  tmpdir: String, path of a temporary directory to write mock data to.
  bucket_uri: String URI of cloud storage bucket to upload mock data to.

Returns:
  stderr: String output from running the gsutil command to upload mock
            data.
zjs,htmlr  r  r   -jtest*TrV  rf  )r   r   r   r   r   r   ru  )r   r   extension_list_stringr   r   s       r   _upload_test_dataQTestCp.test_gzip_transport_encoded_upload_and_download.<locals>._upload_test_data,	  sr     
	!	! ) *~~
d1
'',,vw
'
z

 -1  2f !!*a0mr   c                    > U u  p#nSnTR                  UR                  U5      U5        TR                  UR                  U5      U5        TR                  UR                  U5      U5        g)zEnsure the correct files were marked for compression.

Args:
  local_uris: Tuple of three strings; each is the file path to a file
              containing mock data.
  stderr: String output from running the gsutil command to upload mock
          data.
z2Using compressed transport encoding for file://{}.N)r   rZ  r  )r  r   local_uri_htmllocal_uri_jslocal_uri_txtassert_base_stringr   s         r   _assert_sent_compressedWTestCp.test_gzip_transport_encoded_upload_and_download.<locals>._assert_sent_compressedD	  se     5?1nMO
mm&--n=vF
mm&--l;VD
)00?Hr   r  r  c                   > [        U S5      n[        U S5      n[        U S5      nT	R                  5       nX#U4 Hy  nT	R                  SU/SS9nT	R                  US5        T	R                  SU[        U5      /5        [	        US	5       nT	R                  UR                  5       U5        S
S
S
5        M{     g
! , (       d  f       M  = f)zEnsure the files are not compressed when they are stored in the bucket.

Args:
  bucket_uri: String with URI for bucket containing uploaded test data.
  contents: Byte string that are stored in each file in the bucket.
r  r  r  r   Tr  r  r   r   N)r   r   r   r   r   r   r   )
r   r   r  r  r  r  rN  r  r   r   s
            r   _assert_stored_uncompressed[TestCp.test_gzip_transport_encoded_upload_and_download.<locals>._assert_stored_uncompressedS	  s     J4n*i0l:z2m""$f >#TBF$?@c4<01&$1


1668X
.  	 ?  s   
!B88
C	Nr  )	r   r  r  r  r  r   r   r  r   s	   `        r   /test_gzip_transport_encoded_upload_and_download6TestCp.test_gzip_transport_encoded_upload_and_download		  sJ    .00I :> /( &7%8"J
v2FJ/
+r   c                 ^   U R                  5       nSnU R                  SUS9n[        SS[        [        5      4/5         U R                  SSSS	S
U[        U5      /SS9nU R                  US5        U R                  (       d  U R                  SU5        SSS5        g! , (       d  f       g= f)zTest non resumable, gzip encoded files upload correctly in parallel.

This test generates a small amount of data (e.g. 100 chars) to upload.
Due to the small size, it will be below the resumable threshold,
and test the behavior of non-resumable uploads.
sd   xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
   r  r   r
  r9  r  rX  r   -Jr  TrV  4send: Using gzip transport encoding for the request.N)
rs  r  r4   r   r[   r   r   ru  r   r   r   r   r   r   r   s        r   9test_gzip_transport_encoded_parallel_upload_non_resumable@TestCp.test_gzip_transport_encoded_parallel_upload_non_resumablel	  s     ""$JH2AF	*?WN   
!~~tT4
  f
 !!*b1%%L	
! 
! 
!s   AB
B,c                    U R                  5       n[        U R                  S9nU R                  SUS9n[	        SS[        [        5      4/5         U R                  SSSS	S
U[        U5      /SS9nU R                  US5        U R                  (       d  U R                  SU5        SSS5        g! , (       d  f       g= f)a  Test resumable, gzip encoded files upload correctly in parallel.

This test generates a large amount of data (e.g. halt_size amount of chars)
to upload. Due to the large size, it will be above the resumable threshold,
and test the behavior of resumable uploads.
sizer  r  r
  r9  r  rX  r   r  r  TrV  r  N)rs  rW   r  r  r4   r   r[   r   r   ru  r   r   r  s        r   5test_gzip_transport_encoded_parallel_upload_resumable<TestCp.test_gzip_transport_encoded_parallel_upload_resumable	  s     ""$J%4>>:H2AF	*?WN   
!~~tT4
  f
 !!*b1%%L	
! 
! 
!s   AB//
B=c                    U R                  5       nSS-  nU R                  5       nU R                  SX2S9  U R                  SX2S9  U R                  SX2S9  U R                  SX2S9  U R                  SS	[        R
                  R                  US
5      [        U5      /5        U R                  US5        [        US5      n[        US5      n[        US5      n[        US5      nU R                  SU/SS9nU R                  US5        U R                  SU/SS9nU R                  US5        U R                  SU/SS9nU R                  US5        U R                  SU/SS9nU R                  US5        U R                  5       n	XEXg4 HU  n
U R                  SU
[        U	5      /5        [        U	S5       nU R                  UR                  5       U5        S S S 5        MW     g ! , (       d  f       Mi  = f)Nr  r  r  r  r  r  testr   r  r  rg  r   Tr  r  r   )rs  r  r   r   r   r   r   r   ru  rY  r   r   r   )r   r   r   r   r  r  r  uri4r  r  rN  r   s               r   !test_gzip_all_upload_and_download(TestCp.test_gzip_all_upload_and_download	  s   ""$Je|H!F+fP)FN*VO&KNN	t	fg	&	j		 	
A.
K(D
I&D
J'D
F#D^^VTN$^?FV89^^VTN$^?FV89^^VTN$^?FV89^^VTN$^?FV89  "FD'
nndCf./8,  (s   4!G""
G1	c           	      <   U R                  5       nSS-  nU R                  5       nU R                  SUUS9nU R                  SUUS9nU R                  SSS[        R
                  R                  US	5      [        U5      /S
S9nU R                  US5        U R                  SU-  U5        U R                  SU-  U5        U R                  (       d  U R                  SU5        [        US5      n[        US5      nU R                  5       n	Xx4 Hy  n
U R                  SU
/S
S9nU R                  US5        U R                  SU
[        U	5      /5        [        U	S5       nU R                  UR                  5       U5        SSS5        M{     g! , (       d  f       M  = f)zTest gzip encoded files upload correctly.

This checks that files are not tagged with a gzip content encoding and
that the contents of the files are uncompressed in GCS. This test uses the
-J flag to target all files.
r  r  r  r  r  r  r   r  r  TrV  rg   z2Using compressed transport encoding for file://%s.r  r   r  r  r   N)rs  r  r   r   r   r   r   r   ru  r   r   r   r   r   r   )r   r   r   r   
local_uri1
local_uri2r   remote_uri1remote_uri2r  rN  r  r   s                r   3test_gzip_transport_encoded_all_upload_and_download:TestCp.test_gzip_transport_encoded_all_upload_and_download	  s    ""$Je|H!F$$z,2.6 % 8J $$v,2.6 % 8J ^^	tT	fg	&	j		 	  F
 	
A.MM<
K 	MM<
K ##
mmJ z:.Kz6*K  "F)~~vsm4~@f
&"=>
nndCf./8, 	 * s   !F
F	c                     / SQ/ SQ4nU R                   (       a
  [        u  p#nOSnSnSnU H6  nU R                  USUS9nU R                  X65        U R                  XF5        M8     g)	z)Test that mixing compression flags error.)r   r  r  html, jsa.jsb.js)r   r  r   r  r  r  r   CommandExceptionz:Specifying both the -z and -Z options together is invalid.TrG  Nr   )_GCLOUD_STORAGE_GZIP_FLAG_CONFLICT_OUTPUTr   r   r   casesr   expected_error_prefixexpected_error_substringcaser   s          r   test_both_gzip_options_error#TestCp.test_both_gzip_options_error	  s     	76	8E 
3 Go.F o0
F ~~d,0.=  ?f mm)2
mm,5 r   c                     / SQ/ SQ4nU R                   (       a
  [        u  p#nOSnSnSnU H6  nU R                  USUS9nU R                  X65        U R                  XF5        M8     g)	z0Test that mixing transport encoding flags error.)r   r  r  r   r  r  )r   r  r   r  r  r  r   r  z:Specifying both the -j and -J options together is invalid.TrG  Nr  r  s          r   /test_both_gzip_transport_encoding_options_error6TestCp.test_both_gzip_transport_encoding_options_error
  s     	76	8E 
3 Go.F o0
F  ~~d,0.=  ?f mm)2
mm,5 r   c                     / SQ/ SQ/ SQ/ SQ4nU R                   (       a
  [        u  p#nOSnSnSnU H6  nU R                  USUS	9nU R                  X65        U R                  XF5        M8     g
)z@Test that mixing transport encoding and compression flags error.)r   r  r  r   r  r  )r   r  r  r   r  r  )r   r  r   r  r  r  )r   r  r   r  r  r  r   r  z@Specifying both the -j/-J and -z/-Z options together is invalid.TrG  Nr  r  s          r    test_combined_gzip_options_error'TestCp.test_combined_gzip_options_error&
  s     !O;=E
 
3 Go.F o0
L  ~~d,0.=  ?f mm)2
mm,5 r   c                     U R                  SS9nU R                  5       nSUS S -  nU R                  SSU[        U5      /SS9nU R	                  S	U5        U R                  US
5        g )N)tmpxyzrD  z%s*r   ra  TrV  Copying file:r   )r   rs  r   r   r   ru  )r   r|  r   wildcard_urir   s        r   /test_upload_with_subdir_and_unexpanded_wildcard6TestCp.test_upload_with_subdir_and_unexpanded_wildcard=
  sy      +A BF""$J6#2;&L^^	t\4
#34D  JFMM/6*
A.r   c           
         U R                  SS9nU R                  5       n[        S/5         U R                  SSSU[	        U5      /SS9nU R                  S	U5        S S S 5        U R                  US
5        g ! , (       d  f       N!= f)Nr  r  r
  check_hashesneverr  zContent-MD5: invalid-md5r   TrV  r  r   )r   rs  r4   r   r   r   ru  )r   r|  r   r   s       r   Btest_upload_does_not_raise_with_content_md5_and_check_hashes_neverITestCp.test_upload_does_not_raise_with_content_md5_and_check_hashes_neverF
  s      E 3F""$J	BC	D~~+T6
  f mmOV, 
E 	
A. 
E	Ds   1A88
Bc                    U R                  5       nU R                  5       nU R                  USSS9  U R                  USSS9  U R                  US5        U R	                  SS[        U5      U/5        [        [        R                  R                  XR                  S	S
5      5       nU R                  S
SR                  UR                  5       5      5        SSS5        g! , (       d  f       g= f)z8Tests that cp works with object names ending with slash.zabc/s   dirr  zabc/defr   rg   r   ra  abcdefrf  N)r  rs  r   ru  r   r   r   r   r   r   r   r   r  )r   r   r   r   s       r    test_cp_object_ending_with_slash'TestCp.test_cp_object_ending_with_slashQ
  s     !F""$J"(%  ' 	"+%  ' 	
A.NND$Z 0&9: 
bggll6#9#95%H	IQ
udii67 
J	I	Is   '0C  
C.c                 D   U R                  5       nU R                  USS9nU R                  US5        U R                  S:X  a  SnOSnU R	                  5          U R                  S[        U5      S/S	SS
9nSSS5        U R                  WU5        g! , (       d  f       N!= f)z6Tests that cp fails without read access to the object.r   rq  r   r  AccessDeniedzAnonymous \S+ do(es)? not haver   r  TrG  N)rs  r   ru  r   SetAnonymousBotoCredsr   r   rY  )r   r   
object_uriexpected_error_regexr   s        r   test_cp_without_read_access"TestCp.test_cp_without_read_accessc
  s    
 ""$J""j6"JJ 	
A.$,>		#	#	%~~tT*%5u=,0./  1f 
& 	V12	 
&	%s   B
Bc           	         U R                  5       nU R                  5       nU R                  USS9  [        R                  R                  US5      n[        R                  " U5        [        R                  " [        R                  R                  US5      5        [        R                  " [        R                  R                  US5      [        R                  R                  US5      5        [        R                  " [        R                  R                  US5      5        U R                  SSSU[        U5      /5        g)	z4Tests that cp -e -r ignores symlinks when recursing.r   r  subdirmissingr   r  -eN)rs  r  r   r   r   r   r  r  rmdirr   r   )r   r   r   r1  s       r   test_cp_minus_r_minus_eTestCp.test_cp_minus_r_minus_ey
  s     ""$J!F 	v7WW\\&(+FHHVHHRWW\\&),- JJrww||FI.VY0OPHHRWW\\&),-NND$fd:.>?@r   c                    U R                  5       nU R                  US9n[        R                  R	                  US5      nU R                  5       n[        R                  " X#5        U R                  SSSU< [        R                  R                  < S3[        US5      /SS	9nU R                  S
U5        U R                  (       a  U R                  SU5        OU R                  SU5        U R                  SSSX#[        US5      /SSS9nU R                  S
U5        U R                  (       a(  U R                  SU5        U R                  SU-  U5        g U R                  SU5        U R                  SU-  U5        g )Nr  
cp_minus_erX  r   r3  rb  filesTrV  zCopying filezSkipping symlinkzSkipping symbolic linkr  r   rG  z#URL matched no objects or files: %sz%CommandException: No URLs matched: %s)r  r   r   r   r   rs  r  r   sepr   r   r   )r   	fpath_dirr|  r  r   r   s         r   test_cp_minus_eTestCp.test_cp_minus_e
  sZ   ""$I  	 2FWW\\)\2F""$JJJv
 ^^dDbggkk*Z!
 +/  0F 	MM.&)
mm&/
mm,f5
 ^^	tT6	j'	"	$	  F
 	MM.&)
mm&/
mm9FBFK
mm,f5
mm;fDfMr   c                     SnU R                  US9nU R                  5       nU< [        R                  < S3nU R	                  SSU[        U5      /5        U R                  X15        g)z'Tests that cp -m works with a wildcard.r  r  rb  rX  r   N)r  rs  r   r:  r   r   ru  )r   num_test_filestmp_dirr   r  s        r   test_cp_multithreaded_wildcard%TestCp.test_cp_multithreaded_wildcard
  s^    N  N ;G""$J%rvv.LNND$d:.>?@
;r   c                 b   SnU R                  SUS9nU R                  5       nU R                  SS[        U5      [        U5      U/5        [	        [
        R                  R                  US5      S5       nUR                  5       nU R                  XQ5        SSS5        g! , (       d  f       g= f)z@Tests that cp -m works when a source argument is provided twice.s   edger  )r   r   rX  r   r   N)
r   r  r   r   r   r   r   r   r   r   )r   r  r,  r@  in_fpr   s         r   test_cp_duplicate_source_args$TestCp.test_cp_duplicate_source_args
  s     O""u"OJ  "GNND$Z 0$z2BGLM	bggll7E*D	1Uh
x1 
2	1	1s   5"B  
B.z<gsutil doesn't support S3 customer-supplied encryption keys.c                 f   U R                   [        R                  :X  a  [        R                  " S5      $ SnU R                  SU[        S9nU R                  5       nSS[        4/n[        U5         U R                  S[        U5      [        U5      /5        SSS5        [        US	5       nU R                  UR                  5       U5        SSS5        U R                  5       nSS[        4SS
[        4SS[        4/n[        U5         U R                  S[        U5      [        U5      /5        SSS5        [        US	5       nU R                  UR                  5       U5        SSS5        g! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       Nf= f! , (       d  f       g= f)z&Tests downloading an encrypted object.3gsutil does not support encryption with the XML APIrT  r  r   r   encryption_keyr
  rJ  r   Nr   decryption_key1decryption_key2)test_apir   XMLr;   r  r   r7   r   r4   r   r   r   r   r   r:   r9   )r   r  r,  r   r<  r   r  boto_config_for_test2s           r   !test_cp_download_encrypted_object(TestCp.test_cp_download_encrypted_object
  sr    }}']]
?A AO""u,;2F # HJ !E%'79MNO	2	3
nndD,d5k:; 
4	eT	a
qvvx1 
   "F	#%9:	$&:;	$&:; 
3	4
nndD,d6l;< 
5	fd	q
qvvx1 
	 
4	3		 
5	4		s0   1'E/,!F 
'F!F"/
E= 
F
F"
F0c                 8   U R                   [        R                  :X  a  [        R                  " S5      $ SnU R                  SU[        S9nU R                  5       nU R                  S[        U5      [        U5      /SSS9nU R                  S	[        -  U5        g
)z@Tests downloading an encrypted object without the necessary key.rH  rT  r  rI  r   r   Tr   z*Missing decryption key with SHA256 hash %sN)rM  r   rN  r;   r  r   r7   r   r   r   r   r8   )r   r  r,  r   r   s        r   -test_cp_download_encrypted_object_without_key4TestCp.test_cp_download_encrypted_object_without_key
  s     }}']]
?A AO""u,;2F # HJ !E^^	tJe-  F 	MM4'	()/1r   c                    U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       n[        US5      nSnU R                  USS9nSS[        4/n[        U5         U R                  S[        U5      [        U5      /5        SSS5        U R                  U[        5        [        U5         U R                  5       nU R                  S[        US5      [        U5      /5        [        US	5       nU R                  UR                  5       U5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       N(= f! , (       d  f       g= f)
z$Tests uploading an encrypted object.rH  r  rT  r   rD  r
  rJ  r   Nr   )rM  r   rN  r;   r  rs  r   r   r7   r4   r   AssertObjectUsesCSEKr   r   r   )r   r   r,  r  r   r<  r  r   s           r   test_cp_upload_encrypted_object&TestCp.test_cp_upload_encrypted_object
  s.    }}']]
?A A""$Jj%(JM%HE%'79MNO 
2	3
nndDKj)9:; 
4 	j*>?	2	3""$f
nndDU3T&\BC=1 	 
4	3 
4	3 	 
4	3s1   8'D>AE !E-E >
E
E	E  
E.z1No resumable upload or encryption support for S3.c                    U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       n[        US5      nU R                  SU R                  -  S9nSS[        [        5      4SS[        4/nU R                  [        R                  " [        SS	5      5      S9n[        U5         U R!                  S
SXSU/SSS9nU R#                  SU5        U R!                  S
X2/SS9nU R#                  SU5        U R!                  SU/SS9n[%        US5       nU R#                  ['        U5      U5        SSS5        SSS5        U R)                  U[        5        g! , (       d  f       N-= f! , (       d  f       N6= f)z@Tests that an encrypted upload resumes after a connection break.rH  r  r  rS  r
  r9  rJ  Tr  r   r  r   r   Artifically halting uploadrV  Resuming uploadr   r  r   N)rM  r   rN  r;   r  rs  r   r   r  r   r[   r7   r  r  r(   r4   r   r   r   rI   rW  )	r   r   object_uri_strr   r<  r  r   r  fps	            r   /test_cp_resumable_upload_encrypted_object_break6TestCp.test_cp_resumable_upload_encrypted_object_break  s    }}']]
?A A""$J*e,N)>?E%'<c'lK%'79MNP,,8qAB - D 
2	3~~
$&8 /0,0	  2f
 mm0&9~~tU;4~Pf
mm%v.~~v~6d~Kft8<fE  
4 	n.BC  
4	3s%    A*E>*E-E>-
E;	7E>>
Fc                    U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       n[        US5      nSU R                  -  nU R                  US9nSS[        [        5      4SS[        4/nU R                  [        R                  " [        SS	5      5      S9n[        U5         U R!                  S
SXdU/SSS9nU R#                  SU5        SSS5        SS[        [        5      4SS[$        4SS[        4/n[        U5         U R!                  S
XB/SS9nU R#                  SU5        SSS5        U R'                  U[        5        g! , (       d  f       N= f! , (       d  f       N6= f)zCTests that an encrypted upload resume uses original encryption key.rH  r  r  rS  r
  r9  rJ  Tr  r   r  r   r   r[  NrK  rV  r\  )rM  r   rN  r;   r  rs  r   r  r   r   r[   r7   r  r  r(   r4   r   r   r9   rW  	r   r   r]  r  r   r<  r  r   rO  s	            r   7test_cp_resumable_upload_encrypted_object_different_key>TestCp.test_cp_resumable_upload_encrypted_object_different_key3  s    }}']]
?A A""$J*e,N4>>)M7E%'<c'lK%'79MNP,,8qAB - D 
2	3~~
$&8 /0,0	  2f
 mm0&9 
4 '(=s7|L&(924&(8:NOQ
 
3	4~~tU;4~Pf
mm%v. 
5
 	n.BC' 
4	3 
5	4s   (E$ %E5$
E25
Fc                 D   U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       n[        US5      nSU R                  -  nU R                  US9nSS[        [        5      4SS[        4/nU R                  [        R                  " [        SS	5      5      S9n[        U5         U R!                  S
SXdU/SSS9nU R#                  SU5        SSS5        SS[        [        5      4SS[$        4/n[        U5         U R!                  S
XB/SS9nU R'                  SU5        U R#                  SU5        U R#                  SU5        U R)                  U[$        5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)zDTests that an encrypted upload does not resume without original key.rH  r  r  rS  r
  r9  rJ  Tr  r   r  r   r   r[  NrV  r\  z%does not match current encryption keyzRestarting upload from scratch)rM  r   rN  r;   r  rs  r   r  r   r   r[   r7   r  r  r(   r4   r   r   r9   r  rW  rb  s	            r   5test_cp_resumable_upload_encrypted_object_missing_key<TestCp.test_cp_resumable_upload_encrypted_object_missing_keyW  s    }}']]
?A A""$J*e,N4>>)M7E%'<c'lK%'79MNP,,8qAB - D 
2	3~~
$&8 /0,0	  2f
 mm0&9 
4 '(=s7|L&(8:NOQ 
3	4~~tU;4~Pf
(&1
mm;VD
mm4f= 0DE 
5	4 
4	3 
5	4s   (F AF 
F
Fc                 L    U R                  SU/SS9nU R                  SU5        g)z9Strongly consistent check that the object is unencrypted.r   Tr  zEncryption KeyN)r   r  )r   r]  r  s      r   _ensure_object_unencrypted!TestCp._ensure_object_unencrypted{  s*    ^^V^4D^IF%v.r   z#No resumable upload support for S3.c           
         U R                  5       nU R                  SU R                  -  S9nSS[        [        5      4nU R                  [
        R                  " [        SS5      5      S9n[        U/5         U R                  SSXB[        U5      /S	SS
9nU R                  SU5        U R                  SU[        U5      /SS9nU R                  SU5        SSS5        g! , (       d  f       g= f)z=Tests that an upload can be resumed after a connection break.r  rS  r
  r9  Tr  r   r  r   r   r[  rV  r\  N)rs  r   r  r   r[   r  r  r(   r4   r   r   r   r   r   r   r<  r  r   s         r   test_cp_resumable_upload_break%TestCp.test_cp_resumable_upload_break  s     ""$J)>?E$&;S\J,,8qAB - D 
34	5~~
$&8
z
 /0,0  2f mm0&9~~tUD,<=,0  2f
mm%v. 
6	5	5s   8AC  
C.c                    U R                  5       n[        U R                  S9nU R                  SUS9nSS[	        [
        5      4nU R                  [        R                  " [        SS5      5      S9n[        U/5         U R                  S	S
SSXS[        U5      /SSS9nU R                  SU5        U R                  SU5        U R                  S	S
SU[        U5      /SS9nU R                  SU5        U R                  SU5        SSS5        U R                  5       n[        US5      nU R                  SU/SS9n	U R                  U	S5        U R                  S
U[        U5      /5        [        US5       n
U R                  U
R!                  5       U5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)z0Tests that a gzip encoded upload can be resumed.r  r  rC  r
  r9  Tr  rS  r  r   r  r  r   r   r  r[  rV  r\  Nr   r  r  r   )rs  rW   r  r   r   r[   r  r  r(   r4   r   r   r   r   r   r   r   )r   r   r   	local_urir<  r  r   temp_uri
remote_urir  r   s              r   +test_cp_resumable_upload_gzip_encoded_break2TestCp.test_cp_resumable_upload_gzip_encoded_break  s    ""$J%4>>:H##j8#LI$&;S\J,,8qAB - D 
34	5~~
d02D
z
 /0,0  2f mmJ
mm0&9~~tT4#J/1,0  2f mm%v.
mmJ! 
6( ""$Hj*-J^^VZ0^EF ;<NND*d8n56	h	
qvvx* 
	3 
6	52 
	s   ?BF!7!F2!
F/2
G c                    U R                  5       nU R                  SU R                  -  S9nU R                  [        R
                  :X  a=  U R                  [        R                  " [        S[        R                  S5      5      S9nO<U R                  [        R                  " [        S[        R                  S5      5      S9nSS[        [        5      4n[        U/5         U R!                  SS	S
X2[#        U5      /SS9nU R                  [        R
                  :X  a  U R%                  SU5        OU R%                  SU5        SSS5        g! , (       d  f       g= f)z7Tests that a resumable upload completes with one retry.r  rS  r  )unusedrv  rv  rv  r
  r9  r  r   r  r   rV  zGot retryable failureRetryingN)rs  r   r  rM  r   rN  r  r  r0  r^   BadStatusLiner   BadStatusCodeErrorr   r[   r4   r   r   r   )r   r   r   r  r<  r   s         r   test_cp_resumable_upload_retry%TestCp.test_cp_resumable_upload_retry  s@    ""$J)>?E }}'..
&q+*C*C F 9.   ..
&$77 :DE9F. G %&;S\J	34	5~~
*,>
z
 -.	  /f
 
+//	)-v6j&) 
6	5	5s   "A"E
Ec                    U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       nU R                  [        R                  " [        S[        R                  S5      5      S9nSS[        S[        -  5      4S/n[        U5         U R                  S	S
SUS[!        US5      /S[        -  SS9nU R#                  SU5        SSS5        g! , (       d  f       g= f)zATests that a streaming resumable upload completes with one retry.z1XML does not support resumable streaming uploads.r  rw  rS  r
  r  r  )Botor:  2r  r   r  r  r     aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar   r  rx  N)rM  r   rN  r;   r  rs  r   r  r  r0  r   rz  r   r[   r4   r   r   r   )r   r   r  boto_configs_for_testr   s        r   (test_cp_resumable_streaming_upload_retry/TestCp.test_cp_resumable_streaming_upload_retry  s     }}']]NOO""$J,,fll$Q(;(N(N%C	E7F, G
 '(C!#-023OQ	3	4~~
*,>
z5
! %.$7,-  /f mmJ' 
5	4	4s   #;C''
C5z'preserve_acl flag not supported for S3.c           
         U R                  5       nU R                  USS9nU R                  SSSS[        U5      /5        U R                  SSSS[        U5      /5        U R	                  5          U R                  SS	[        U5      [        US
5      /SSS9nU R                  SU5        S S S 5        g ! , (       d  f       g = f)Nr   rq  r  chz-uz
AllUsers:Rz
AllUsers:Wr   z-pr  Tr   rG  z0OWNER permission is required for preserving ACLs)rs  r   r   r   r+  r   )r   r   r,  r   s       r   test_cp_preserve_no_owner TestCp.test_cp_preserve_no_owner  s    ""$J""j6"JJ 	NNE4|T*5EFGNNE4|T*5EFG		#	#	%~~tJ'
E"$	  f
 mmFO 
&	%	%s   /;B33
Cc                    U R                  5       n[        S5      nUS-   U-   nU R                  S[        -  SS9nSS[	        [
        5      4n[        U/5         U R                  SU[        U5      /S	S
9nU R                  SUR                  U5      5        S S S 5        SS[	        S[        -  5      4n[        U/5         U R                  SU[        U5      /S	S
9nU R                  SUR                  U5      5        S S S 5        U R                  S[        US5      U/S	S
9nU R                  SUR                  U5      5        g ! , (       d  f       N= f! , (       d  f       N^= f)N   r   r  r  rV  r
  r9  r   TrV  r   rg   )rs  r@   r   r\   r   r[   r4   r   r   r   ru  )r   r   final_size_stringfinal_progress_callbackr   r<  r   s          r   test_cp_progress_callbacks!TestCp.test_cp_progress_callbacks  sb   ""$J/8/#58II5IE$&;S\J	34	5~~tUD,<=,0  2f
q&,,'>?@ 
6 %&;SW=MN	34	5~~tUD,<=,0  2f
q&,,'>?@ 
6 ^^T4
E#:EB*.  0FQ%<=> 
6	5
 
6	5s   >E >E
E
E$c                    U R                  5       nU R                  SU R                  -  S9nSS[        [        5      4n[        U/5         U R                  SU[        U5      /5        SSS5        g! , (       d  f       g= f)z;Tests that a basic resumable upload completes successfully.r  rS  r
  r9  r   N)rs  r   r  r   r[   r4   r   r   )r   r   r   r<  s       r   test_cp_resumable_uploadTestCp.test_cp_resumable_upload  sp     ""$J)>?E$&;S\J	34	5
nndE4
#345 
6	5	5s   A22
B c                 p   U R                  5       nU R                  SSU R                  -  S9nSS[        [        5      4n[        U/5         [        [        [        US5      5      [        R                  U R                  5      nU R                  [        R                  " [        SS5      5      S9n U R                  S	S
XR[        US5      /SSS9nU R!                  SU5        U R#                  [$        R&                  R)                  U5      SU-  5        [$        R*                  S:X  aX  [-        [.        R0                  " [$        R.                  " U5      R2                  5      5      nU R5                  [-        S5      U5        [7        U5         SSS5        g! [7        U5        f = f! , (       d  f       g= f)z=Tests that a tracker file is created with a resumable upload.r  r  rC  r
  r9  Tr  rS  r   r  r   r   r[  zTracker file %s not present.posixi  N)rs  r   r  r   r[   r4   rE   r   r   rH   UPLOADrM  r  r  r(   r   r   r   r   r   r  r(  octr   S_IMODEst_moder   r=   )r   r   r   r<  tracker_filenamer  r   r   s           r   *test_resumable_upload_break_leaves_tracker1TestCp.test_resumable_upload_break_leaves_tracker!  s    ""$J%$:OPE$&;S\J	34	5+
tJ6
79O9O
--  ..<< :4 CD / F,&(:U#!
 12.2   4 	2F;'7869II	K 77gT\\"''*:";"C"CDE$


3u:t
,*+- 
6	5, 	*+- 
6	5s&   A&F'3CFF'F$$F''
F5c           
      J   U R                  5       nU R                  5       nU R                  SUSU R                  -  S9nU R                  [        R
                  " [        SS5      5      S9nSS[        [        5      4n[        U/5         U R                  S	S
XC[        U5      /SSS9nU R                  SU5        U R                  SUSU R                  -  S-  S9nU R                  S	U[        U5      /SSS9nU R                  SU5        SSS5        g! , (       d  f       g= f)zpTests a resumable upload where the uploaded file changes size.

This should fail when we read the tracker data.
r  r  r  Tr  rS  r
  r9  r   r  r   r   r[  rg   ResumableUploadAbortExceptionN)rs  r  r   r  r  r  r(   r   r[   r4   r   r   r   )r   r   r@  r   r  r<  r   s          r   /test_cp_resumable_upload_break_file_size_change6TestCp.test_cp_resumable_upload_break_file_size_change?  sH    ""$J  "G%'.)-)>   @E ,,8qAB - D %&;S\J	34	5~~
$&8
z
 /0,0  2f mm0&9!!E)0+/$..+@1+D " Fe ~~tUD,<=./,0  2f mm3V< 
6	5	5s   
BD
D"c           
         U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       nU R                  5       nU R                  SUS[        -  [        -  S9nU R                  [        R                  " [        S[        [        5      S-  5      5      S9nSS	[        [        5      4nSS
[        [        S-  5      4n[        XV/5         U R                  SSXC[!        U5      /SSS9nU R#                  SU5        U R                  SUS[        -  [        -  S9nU R                  SU[!        U5      /SSS9nU R#                  SU5        SSS5        g! , (       d  f       g= f)zATests a resumable upload where the uploaded file changes content.zXML doesn't make separate HTTP calls at fixed-size boundaries for resumable uploads, so we can't guarantee that the server saves a specific part of the upload.r  r  r  T   rS  r
  r9  r  r  r   r  r   r   r[  r  #doesn't match cloud-supplied digestN)rM  r   rN  r;   r  rs  r  r   r[   r  r  r(   r   r   r4   r   r   r   r   r   r@  r   r  resumable_threshold_for_testresumable_chunk_size_for_testr   s           r   2test_cp_resumable_upload_break_file_content_change9TestCp.test_cp_resumable_upload_break_file_content_change^  s    }}']])* * ""$J  "G%'.)-')A   CE ,,fll"4#&w<##5	778, 9 %-.C$'L$2 %-/J%(3%7%9!		%E
G~~
$&8
z
 /0,0  2f mm0&9!!E)0+/'>G+C " Ee ~~tUD,<=./,0  2f mm:FC
G 
G 
Gs   $A?E,,
E:c           
         U R                  5       nU R                  5       nU R                  SUS[        -  [        -  S9nU R                  [        R
                  " [        S[        [        5      S-  5      5      S9nSS[        [        5      4nSS	[        [        S
-  5      4n[        XV/5         U R                  SSXC[        U5      /SSS9nU R                  SU5        U R                  SUS[        -  S9nU R                  SU[        U5      /SSS9nU R                  SU5        SSS5        g! , (       d  f       g= f)zeTests a resumable upload where the uploaded file changes content.

This should fail hash validation.
r  r  r  Tr  rS  r
  r9  r  r  r   r  r   r   r[  r  N)rs  r  r   r[   r  r  r(   r   r   r4   r   r   r   r  s           r   0test_cp_resumable_upload_break_file_smaller_size7TestCp.test_cp_resumable_upload_break_file_smaller_size  sn    ""$J  "G%'.)-')A   CE ,,fll"4#&w<##5	778, 9 %-.C$'L$2 %-/J%(3%7%9!		%E
G~~
$&8
z
 /0,0  2f mm0&9!!E)0+/'> " ;e ~~tUD,<=./,0  2f mm3V<
G 
G 
Gs   0A8D11
D?c           	      x   U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       n[        [        US5      5      nSnSnU R                  UUS9n[        U5      n[        U[        R                  U R                   U5      nSn[        U-   R                  [        5      n	[        5       n
U
R!                  U	5        U
R#                  5       nU[$        -   U-   S-   nSnU R'                  UUUS	U [(        S
9n[+        U[-        UR.                  5      5      nU/n[0        n[3        UUUUS9   [5        SSS[-        U5      4SS[(        4/5         U R7                  SU[        US5      /SS9nU R9                  SU5        U R;                  [<        R>                  RA                  U5      SU-  5        U R7                  S[        US5      /SS9nU RC                  UR                  S5      U5        S	S	S	5        [E        U5        g	! , (       d  f       N= f! [E        U5        f = f)z>Tests that an encrypted composite upload resumes successfully.rH  r     foobarfoobarrV  123_0rf  Nr   r   r   rJ  )encryption_key_sha256r
  #parallel_composite_upload_thresholdr  r
  (parallel_composite_upload_component_sizerJ  r   TrV  z/Found 1 existing temporary components to reuse.)Tracker file %s should have been deleted.r  r  r  )#rM  r   rN  r;   r  rs  r   r   r   rE   rH   PARALLEL_UPLOADrF   r   rD   rK   update	hexdigestrG   r   r7   r   r   r  r8   r   r4   r   r   r  r   r   r  r   r=   )r   r   dst_urlr  rD  source_filesrc_urltracker_file_nametracker_prefixencoded_namecontent_md5digestcomponent_object_namecomponent_sizer,  existing_componentexisting_componentsenc_key_sha256r   read_contentss                       r   )test_cp_composite_encrypted_upload_resume0TestCp.test_cp_composite_encrypted_upload_resume  sd    }}']]
?A A""$J"4
E#:;GMI%%}09 & ;K";/G +7+:+J+J+/=='C N 0+=EEdKL(K|$""$F+.LL#$&*+ N""j/D,9/>,J2F # HJ ++@+.z/D/D+EG-.4N"#4#1#69GI
+
@?~ %';
<	! 	 ;Z 78   NGPGGNN,-7:KK	M tJ/F'G59 ' ;--g6F	" )*#	 	" )*s%   !H, (BHH, 
H)%H, ,H9c           	         U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       n[        [        US5      5      nSnU R                  USS9n[        U5      n[        U[        R                  U R                   U5      nSnSnU R                  USS[        S9n	[        U[        U	R                   5      5      n
U
/n["        n[%        XgUUR'                  S	5      5         [)        S
SSS[*        4/5         U R-                  SU[        US5      /SS9nU R/                  SU5        U R1                  SU5        U R3                  [4        R6                  R9                  U5      SU-  5        U R-                  S[        US5      /SS9nU R;                  UR=                  S	5      U5        SSS5        [?        U5        g! , (       d  f       N= f! [?        U5        f = f)zETests that encrypted composite upload restarts given a different key.rH  r  r  rV  r  foo_1r   r  r  r  )r
  r  3r
  rJ  r   TrV  zTdoes not match current encryption key. Deleting old components and restarting uploadz'existing temporary components to reuse.r  r  r  N) rM  r   rN  r;   r  rs  r   r   r   rE   rH   r  r   r7   r   r   r  r8   r   rt  r4   r9   r   r   r  r  r   r   r  r   r   r=   )r   r   r  r  r  r  r  r  existing_component_namer,  r  r  r  r   r  s                  r   *test_cp_composite_encrypted_upload_restart1TestCp.test_cp_composite_encrypted_upload_restart  s    }}']]
?A A""$J"4
E#:;GM%%}%NK";/G +7+:+J+J+/=='C N%""j/6,22F # HJ ++B+.z/D/D+EG-.4N"#4#6#1#8#8#AC+
@
E%';
<! 	
 ;Z 78   N<=C	E 	BFKGGNN,-7:KK	M tJ/F'G59 ' ;--g6F!	& )*'	 	& )*s%   <G' B1GG' 
G$ G' 'G4z$Test uses gs-specific KMS encryptionc                 X   U R                  5       nU R                  SS9n[        US5      n[        5       n[	        SSU4SS/5         U R                  SX#/5        S S S 5        [	        S	/5         U R                  X45        S S S 5        g ! , (       d  f       N5= f! , (       d  f       g = f)
N   abcdrS  composedr
  rJ  r  r
  r  r  r   r
  
prefer_apijsonrs  r   r   r%   r4   r   AssertObjectUsesCMEK)r   r   r   obj_surikey_fqns        r   2test_kms_key_correctly_applied_to_composite_upload9TestCp.test_kms_key_correctly_applied_to_composite_upload  s    ""$J1EJ
+H02G		#W->C 

 nndE,-
 
?@	A
2 
B	A
 
 
B	As   B
/B

B
B)z#No composite upload support for S3.c                 4   U R                  SS9nU R                  SS9n[        US5      n[        SS/5         U R	                  SS	S
X#/5        S S S 5        U R	                  SSU/SS9nU R                  US[        R                  S9  g ! , (       d  f       NA= f)Nr  r  r  rS  r  r  r  r   -snearliner  r  Tr  z Storage class:          NEARLINEflags)rs  r   r   r4   r   assertRegexpMatchesWithFlagsr   
IGNORECASE)r   r   r   r  r  s        r   2test_nearline_applied_to_parallel_composite_upload9TestCp.test_nearline_applied_to_parallel_composite_upload0  s    """<J1EJ
+H	>C 
 nndD*e>?	

 ^^T42$^GF%%f(K.0mm & =
 
s   B		
Bz$chmod on dir unsupported on Windows.c                    U R                  5       n[        [        [        US5      5      [        R
                  U R                  5      n[        R                  R                  U5      nU R                  SS[        -  S9nSS[        [        5      4n[        R                  " U5      R                  n [        R                  " US5        [!        U/5         U R#                  SU[        U5      /SS	S
9nU R%                  SU5        SSS5        [        R                  " X65        [        R                  R'                  U5      (       a  [        R(                  " U5        gg! , (       d  f       N`= f! [        R                  " X65        [        R                  R'                  U5      (       a  [        R(                  " U5        f f = f)z9Tests a resumable upload with an unwritable tracker file.r  r  rC  r
  r9  r   r   r   Tr   Couldn't write tracker fileN)rs  rE   r   r   rH   r  rM  r   r   rU  r   r[   r   r   r  chmodr4   r   r   r  unlink)r   r   r  tracker_dirr   r<  save_modr   s           r   test_cp_unwritable_tracker_file&TestCp.test_cp_unwritable_tracker_fileB  s]    ""$J)T*e457M7M ''//"23K%$.IE$&;S\Jww{#++H
$hh{A!5 67ud:.> ?01.2   4 	4f=	 8 	hh{%	(	)	)
		"# 
* 87 	hh{%	(	)	)
		"# 
*s%   2#E0 0EE0 
E-)E0 0AGc                 2   U R                  S[        -  S9n[        [        [	        U5      5      [
        R                  U R                  5      n[        R                  R                  U5      nU R                  5       n[        R                  " U5      R                  n [        R                  " US5        SS[        [         5      4n[#        U/5         U R%                  S[	        U5      U/5        SSS5        SS[        [        5      4n[#        U/5         U R%                  S[	        U5      U/SS	S
9nU R'                  SU5        SSS5        [        R                  " X55        [        R                  R)                  U5      (       a  [        R*                  " U5        gg! , (       d  f       N= f! , (       d  f       Nq= f! [        R                  " X55        [        R                  R)                  U5      (       a  [        R*                  " U5        f f = f)z0Tests downloads with an unwritable tracker file.r   rS  r   r
  r9  r   Nr   Tr   r  )r   r[   rE   r   r   rH   DOWNLOADrM  r   r   rU  r   r   r  r  r   rX   r4   r   r   r  r  )r   r,  r  r  r   r  r<  r   s           r   (test_cp_unwritable_tracker_file_download/TestCp.test_cp_unwritable_tracker_file_download_  s   
 ""FW,<"=J)T*-.0H0H ''//"23K!Eww{#++H$hh{A&(=s9~N!5 67d:.67 8 '(=s7|L!5 67tJ'7 ?01.2   4 	4f=	 8 	hh{%	(	)	)
		"# 
* 87 87 	hh{%	(	)	)
		"# 
*s=   5G F!1&G 0F2G !
F/+G 2
G <G AHNc           	      .   U R                  5       nSU R                  -  nU R                  USUUS9nU R                  5       nU R                  [        R
                  " [        SS5      5      S9n[        U5         U R                  SSU[        U5      U/S	S
S9nU R                  SU5        [        [        U5      [        R                  U R                  5      n	U R!                  ["        R$                  R'                  U	5      5        U R                  S[        U5      U/S
S9nU R                  SU5        SSS5        [)        US5       n
U R+                  U
R-                  5       US5        SSS5        g! , (       d  f       NE= f! , (       d  f       g= f)zHelper function for different modes of resumable download break.

Args:
  boto_config: List of boto configuration tuples for use with
      SetBotoConfigForTest.
  encryption_key: Base64 encryption key for object encryption (if any).
r  r  r  Fr  rS  r   r  r   Tr   Artifically halting download.rV  Resuming downloadNr   File contents differ)rs  r  r   r   r  r  r(   r4   r   r   r   rE   r   rH   r  rM  r   r   r   isfiler   r   r   )r   boto_configrJ  r   r  r,  r   r  r   r  r   s              r   (_test_cp_resumable_download_break_helper/TestCp._test_cp_resumable_download_break_helper}  s    ""$J4>>)M""j/4,92@ # BJ !E,,8BC - E 
k	*~~
$&8
z
E /0,0  2f mm3V<+,@,G,;,D,D,0MM; oobggnn%567~~tT*%5u=,0  2f
mm'0 
+ 
eT	a
qvvx0FG 
	 
+	* 
	s   :B<E5
"F5
F
Fc                 H    U R                  SS[        [        5      4/5        g)z>Tests that a download can be resumed after a connection break.r
  r9  N)r  r   r[   r  s    r    test_cp_resumable_download_break'TestCp.test_cp_resumable_download_break  s$    11	(#g,73 r   c                     U R                   [        R                  :X  a  [        R                  " S5      $ U R                  SS[        [        5      4SS[        4/[        S9  g)zBTests that an encrypted download resumes after a connection break.rH  r
  r9  rJ  )rJ  N)	rM  r   rN  r;   r  r  r   r[   r7   r  s    r   *test_cp_resumable_encrypted_download_break1TestCp.test_cp_resumable_encrypted_download_break  s`     }}']]
?A A11
)3w<	8
$&:	;	=+ 2 -r   c           	         U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       nSU R                  -  nU R                  USU[        S9nU R                  5       nU R                  [        R                  " [        SS5      5      S9nSS	[        [        5      4SS
[        4/n[        U5         U R!                  SSU[#        U5      U/SSS9nU R%                  SU5        ['        [)        U5      [*        R,                  U R                   5      nU R/                  [0        R2                  R5                  U5      5        SSS5        SS	[        [        5      4SS[        4SS
[6        4/n	[        U	5         U R!                  SS[#        U5      /5        SSS5        SS	[        [        5      4SS
[6        4/n
[        U
5         U R!                  S[#        U5      U/SS9nU R%                  SU5        SSS5        [9        US5       nU R;                  UR=                  5       US5        SSS5        g! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       Ng= f! , (       d  f       g= f)z=Tests that a download restarts with a rotated encryption key.rH  r  r  r  Fr  rS  r
  r9  rJ  r   r  r   Tr   r  NrK  rewritez-krV  zRestarting downloadr   r  )rM  r   rN  r;   r  rs  r  r   r7   r   r  r  r(   r   r[   r4   r   r   r   rE   r   rH   r  r   r   r   r  r9   r   r   r   )r   r   r  r,  r   r  r<  r   r  rO  boto_config_for_test3r   s               r   1test_cp_resumable_encrypted_download_key_rotation8TestCp.test_cp_resumable_encrypted_download_key_rotation  sq    }}']]
?A A""$J4>>)M""j/4,92F # HJ !E,,8BC - E &'<c'lK%'79MNP 
2	3~~
$&8
z
E /0,0  2f mm3V<+,@,G,;,D,D,0MM; oobggnn%567 
4 '(=s7|L&(924&(8:NOQ 
3	4
nnitJ'789 
5
 '(=s7|L&(8:NOQ	3	4~~tT*%5u=,0  2f
mm)62 
5 
eT	a
qvvx0FG 
	; 
4	3$ 
5	4 
5	4 
	s1   BII/I- "I>
I
I*-
I;>
Jc           	      Z   U R                  5       nU R                  USSU R                  -  S9nU R                  5       nU R                  [        R
                  " [        SS5      5      S9nSS[        [        5      4n[        U/5         U R                  S	S
U[        U5      U/SSS9nU R                  SU5        U R                  USSU R                  -  UR                  S9nU R                  S	[        U5      U/SS9nU R                  SU5        SSS5        g! , (       d  f       g= f)zjTests that download restarts the file when the source object changes.

This causes the etag not to match.
r  r  r  Fr  rS  r
  r9  r   r  r   Tr   r  r  r^  rV  r  N)rs  r   r  r   r  r  r(   r   r[   r4   r   r   r   r  r  r   r   r,  r   r  r<  r   s          r   'test_cp_resumable_download_etag_differs.TestCp.test_cp_resumable_download_etag_differs  sK    ""$J""j/4,2T^^,C # EJ !E,,8BC - E$&;S\J	34	5~~
$&8
z
E /0,0  2f mm3V< $$$..(#-#8#8	 % :j
 ~~tT*%5u=,0  2f
*F3% 
6	5	5s   
B	D
D*z%Sliced download requires fast crcmod.z"No sliced download support for S3.c           
      d   U R                  5       nSU R                  -  nU R                  USUS9nU R                  5       nU R                  [        R
                  " [        SS5      5      S9nSS[        U R                  5      4SS	[        U R                  5      4S
/n[        U5         U R                  SSU[        U5      [        U5      /SSS9nU R                  SU5        U R                  US9nU R                  S[        U5      [        U5      /5        U R                  S[        U5      [        U5      /SS9nU R                  SU5        [        US5       n	U R                  U	R                  5       US5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)zCTests that a resumable download restarts if the generation differs.r  r  r  Fr  rS  r
  r9   sliced_object_download_thresholdr
  %sliced_object_download_max_componentsr  r   r  Tr   rG  r  rV   Restarting download from scratchr   r  N)rs  r  r   r   r  r  r(   r   r4   r   r   r   r   r   r   )
r   r   r  r,  r   r  r<  r   identical_filer   s
             r   -test_cp_resumable_download_generation_differs4TestCp.test_cp_resumable_download_generation_differs  s   
 ""$Jdnn,M""j/4,9 # ;J !E,,8BC - E 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm3V< **M*Bn
nndD0$z2BCD~~j!4;
/t  Ef
mm6?t=2HI % 
4	3$ % 
4	3s%   *B3F!"F?F!
F	F!!
F/c           	         U R                  5       nU R                  5       nU R                  USSU R                  -  S9nU R                  [        R
                  " [        SS5      5      S9nSS[        [        5      4n[        U/5         U R                  S	S
U[        U5      U/SSS9nU R                  SU5        [        US-   S5       n[        U R                  S-  5       H  nUR                  S5        M     SSS5        U R                  S	[        U5      U/SSS9nU R!                  SU5        U R                  SU5        SSS5        g! , (       d  f       NX= f! , (       d  f       g= f)zETests download deletes the tracker file when existing file is larger.r  r  r  Fr  rS  r
  r9  r   r  r   Tr   r  _.gstmpr  rg   aNr  zDeleting tracker file)rs  r   r   r  r  r  r(   r   r[   r4   r   r   r   r   r_   r   r  )	r   r   r   r,  r  r<  r   larger_filerm  s	            r   &test_cp_resumable_download_file_larger-TestCp.test_cp_resumable_download_file_larger5  su   ""$J!E""j/4,04>>,A # CJ ,,8BC - E$&;S\J	34	5~~
$&8
z
E /0,0  2f mm3V<	!3';t~~)*A


C
  + ( ~~tT*%5u=./,0  2f *F3
mm+V4 
6	5 (' 
6	5s&   
AE1E<A	E
E	E
E-c                    U R                  5       nU R                  5       nU R                  US9nUS-   n[        US5       nUR	                  S[
        -  5        SSS5        U R                  USS[
        -  S9nU R                  S	S
[        U5      /SS9n[        R                  " SU5      nU R                  US5        U R                  [        UR                  5       5      SS5        UR                  S5      n	[!        [#        U5      [$        R&                  U R(                  5      n
 [        U
S5       nUR	                  U	5        SSS5        SS[+        [
        5      4n[-        U/5         U R                  S[        U5      U/SSS9nU R/                  SU5        U R/                  SU5        U R1                  [2        R4                  R7                  U5      5        U R1                  [2        R4                  R7                  U
5      5        U R1                  [2        R4                  R7                  U5      5        SSS5        [2        R4                  R9                  U
5      (       a  [2        R:                  " U
5        gg! , (       d  f       GN*= f! , (       d  f       GNO= f! , (       d  f       Nn= f! [2        R4                  R9                  U
5      (       a  [2        R:                  " U
5        f f = f)a7  Tests that we do not re-download when tracker file matches existing file.

We only compare size, not contents, so re-download should not occur even
though the contents are technically different. However, hash validation on
the file should still occur and we will delete the file then because
the hashes differ.
r  r  r  abcdNr     efghr  r  r  Tr  \s*ETag:\s*(.*)Could not get object ETagr   "Did not match expected single ETagr
  r9  r   rG  Download already completer  )rs  r  r   r   r   r[   r   r   r   r   r   r  r   rk  groupsr  rE   r   rH   r  rM  r   r4   r   r  r   r   r  r  r  )r   r   r@  r   temp_download_filer^  r,  r  
etag_matchetagr  
tracker_fpr<  r   s                 r   *test_cp_resumable_download_content_differs1TestCp.test_cp_resumable_download_content_differsP  sl    ""$J  "Gw/E*	 #	&"hhv  
' ""j/4,3g,= # ?J ^^T4j)9:$^OF-v6J%@AS**,-q9;AD)*>u*E*9*B*B*.--9$ #&* '&(=s7|L!5 67tJ'7 ?.201   3 	16:<fE(:;<(89:./ 8 
(	)	)
		"# 
*A 
'	&" '& 87 
(	)	)
		"# 
*sI   I=)J2 5J&J2 -CJ!9J2 =
J
JJ2 !
J/+J2 2=K/c                    U R                  5       nU R                  5       nU R                  US9nS[        -  nUS-   n[	        US5       nUR                  U5        SSS5        U R                  USUS9nU R                  SS	[        U5      /S
S9n[        R                  " SU5      n	U R                  U	S5        U R                  [        U	R                  5       5      SS5        U	R                  S5      n
[!        [#        U5      [$        R&                  U R(                  5      n[	        US5       nUR                  U
5        SSS5         SS[+        [        5      4n[-        U/5         U R                  S[        U5      U/S
S9nU R/                  SU5        U R1                  [2        R4                  R7                  U5      5        SSS5        [2        R4                  R9                  U5      (       a  [2        R:                  " U5        gg! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       Nm= f! [2        R4                  R9                  U5      (       a  [2        R:                  " U5        f f = f)z>Tests download no-ops when tracker file matches existing file.r  r  r  r   Nr  r  r  r  Tr  r  r  r   r  r  r
  r9  r   rV  r  )rs  r  r   r[   r   r   r   r   r   r   r   r  r   rk  r  r  rE   r   rH   r  rM  r   r4   r   r  r   r   r  r  r  )r   r   r@  r   matching_contentsr  r^  r,  r  r  r  r  r  r<  r   s                  r   *test_cp_resumable_download_content_matches1TestCp.test_cp_resumable_download_content_matches  s"   ""$J  "Gw/E')*	 $	'2hh ! 
( ""j/4,= # ?J ^^T4j)9:$^OF-v6J%@AS**,-q9;AD)*>u*E*9*B*B*.--9 
	$
t 
%
$&(=s7|L!5 67tJ'7 ?.2   416:(89: 8 
(	)	)
		"# 
*5 
(	' 
%	$ 87 
(	)	)
		"# 
*s=   
H	/H
H= (AH,H= 	
H
H),
H:6H= ==I:c                 8   U R                  5       nU R                  5       nU R                  US[        -  S9nU R	                  USS[        -  S9nU R                  SS[        U5      /SS	9n[        R                  " S
U5      nU R                  US5        U R                  [        UR                  5       5      SS5        UR                  S5      nUS-  n[        [        U5      [         R"                  U R$                  5      n['        US5       n	U	R)                  U5        SSS5         SS[+        [        5      4n
[-        U
/5         U R                  S[        U5      U/SS9nU R/                  SU5        ['        US5       nUR1                  5       nU R                  US[        -  S5        SSS5        U R3                  [4        R6                  R9                  U5      5        SSS5        [4        R6                  R;                  U5      (       a  [4        R<                  " U5        gg! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       Nm= f! [4        R6                  R;                  U5      (       a  [4        R<                  " U5        f f = f)zETests that download overwrites when tracker file etag does not match.r  r  r  r  r  r  r  Tr  r  r  r   z/Did not match regex for exactly one object ETagnonmatchingr  Nr
  r9  r   rV  r  r  efghzQFile not overwritten when it should have been due to a non-matching tracker file.)rs  r  r   r[   r   r   r   r   r   r  r   rk  r  r  rE   r   rH   r  rM  r   r   r   r4   r  r   r  r   r   r  r  r  )r   r   r@  r   r,  r  r  r  r  r  r<  r   rD  r   s                 r   3test_cp_resumable_download_tracker_file_not_matches:TestCp.test_cp_resumable_download_tracker_file_not_matches  s?   ""$J  "Gw79JKE""j/4,3g,= # ?J ^^T4j)9:$^OF-v6J%@AS**,-qFHADMD)*>u*E*9*B*B*.--9 
	$
t 
%$&(=s7|L!5 67tJ'7 ?.2   4,f5%ZZ\(


(45  	(89: 8 
(	)	)
		"# 
*# 
%	$  87 
(	)	)
		"# 
*sH   H(*I ;I+H:.6I$I (
H7:
I	I
II =Jc                     U R                  5       nU R                  SSS9nU R                  SSSS[        U5      [        US5      /5        U R                  S[        US5      U/5        g	)
ATests that upload and download of a doubly-gzipped file succeeds.looks-zipped.gzr   rC  r  content-type:application/gzipr   r  r  N)rs  r   r   r   )r   r   r   s      r   test_cp_double_gzipTestCp.test_cp_double_gzip  sn    ""$J*;fMENN-tTUZ 
 	NND$z5159:r   c                    U R                  5       nU R                  SSS9nU R                  SSSSS[        U5      [        US	5      /S
S9nU R                  (       a%  U R                  SU5        U R                  SU5        O$U R                  SU5        U R                  SU5        U R                  S[        US	5      U/5        g)r,  r-  r   rC  r1  r  r.  r   r  r  TrV  zb'Content-Encoding': b'gzip'z!"contentType": "application/gzip"z'Content-Encoding': 'gzip'zcontentType: 'application/gzip'Nr{  )r   r   r   r   s       r   %test_cp_double_gzip_transport_encoded,TestCp.test_cp_double_gzip_transport_encoded  s    
 ""$J*;fME^^t4dDUZ
 +/  0F 
mm6?
mm7@
mm4f=
mm7@NND$z5159:r   z%TODO(b/293885158) Timeout on Windows.c           
      n  ^ ^ T R                  5       m[        R                  " S5        [        [	        T R
                  5       Vs/ s H'  n[        R                  " [        R                  5      PM)     sn5      R                  S5      n[        R                  " 5         T R                  SUS9nT R                  SSS[        U5      [        T5      /5        [        [        SS	S
9UU 4S j5       nU" 5       nT R                  UT R
                  S5        T R                  5       nT R                  [         R"                  " [%        SS5      5      S9nSS[        [&        5      4n[)        U/5         T R                  SSU[        T5      [        U5      /SS	S9n	T R+                  SU	5        T R+                  SU	5        [-        [.        R0                  R3                  SS[4        5      5      n
[7        U5      U
:  =(       a    U
S:  =(       a
    [9        5       nU(       a  [:        R<                  nO[:        R>                  n[A        [C        U5      UT RD                  5      nT RG                  [H        RJ                  RM                  U5      5        T RG                  [H        RJ                  RM                  SU-  5      5        T R                  S[        T5      [        U5      /SS9n	T R+                  SU	5        [O        US5       nT RQ                  URS                  5       US5        SSS5        T RU                  [H        RJ                  RM                  U5      5        T RU                  [H        RJ                  RM                  SU-  5      5        SSS5        gs  snf ! , (       d  f       N{= f! , (       d  f       g= f)zDTests that download can be resumed successfully with a gzipped file.r   r  zunzipped.txtrC  r   r  txtrf  r   r  c                    > TR                  S[        T5      /SS9n [        R                  " SU 5      nTR	                  US5        TR                  [        UR                  5       5      SS5        [        UR                  S5      5      $ )NduTr  z
(\d+)\s+.*zCould not get object sizer   z0Did not match regex for exactly one object size.)
r   r   r   r   r  r   rk  r  longr  )r  
size_matchr,  r   s     r   _GetObjectSize>TestCp.test_cp_resumable_download_gzip.<locals>._GetObjectSize  sz    ~~tT*%56d~Kf99]F3j
:'BC
s:,,./IK*""1%&&r   zCompresed object size was not large enough to allow for a halted download, so the test results would be invalid. Please increase the compressed object size in the test.Fr  rS  r
  r9  r  TrG  r  z!Downloading to temp gzip filenamer  z	%s_.gztmprV  r  r   File contents did not match.N)+r   randomseedr   r_   r  choicestringascii_lettersr   r   r   r   rU   rm  rt  r  r  r(   r[   r4   r   rY   r  r  r   r   rk  rB   rH   SLICED_DOWNLOADr  rE   r   rM  r   r   r   r  r   r   r   r  )r   rm  r   r|  r:  object_sizer  r  r<  r   sliced_download_thresholdsliced_downloadtrackerfile_typer  r   r,  s   `              @r   test_cp_resumable_download_gzip&TestCp.test_cp_resumable_download_gzip  s!    ""$J
KKN5:4>>5J5Jf**+5J vg  KKM  >H MFNND$tF|T*5EFG >3' 4' !"KT^^	#$   "F,,8BC - E %&;S\J	34	5~~
$&8
z

v,
 -1./  1f mm3V<
mm7@ #7
++//($FBD#E X)BB 02Q60-/  
*::*33+,@,H,<dmmM
 oobggnn%567
oobggnn[6%9:;~~j!4<
0  Ff
mm'08-KL 
rww~~&678
rww~~kF&:;<I 
6	5;| C 
6	5s,   .N.FN&>"N A'N&
N#	N&&
N4c                     [        U S5      (       d+  [        R                  " SS5      nU R                  US9U l        U R                  $ )Ntest_data_favicon_filerB  ztests/test_data/favicon.ico.gzrS  )hasattrrE  rF  r   rJ  )r   r   s     r   _GetFaviconFileTestCp._GetFaviconFile6  sG     4122!!'+KLh$($7$7$7$Jd!&&&r   c                    U R                  5       nU R                  USS9nU R                  5       nU R                  SSSSS[	        U5      [	        U5      /5        [
        R                  " U5       nS[        5       0n[        R                  " XE5        US   R                  5       nSSS5        U R                  5       nU R                  S[	        U5      [	        U5      /5        [        US	5       nS[        5       0n[        R                  " XE5        US   R                  5       nSSS5        U R                  WW5        g! , (       d  f       N= f! , (       d  f       N2= f)
a!  Tests chunked transfer encoded download handling.

Tests that download works correctly with a gzipped chunked transfer-encoded
object (which therefore lacks Content-Length) of a size that gets fetched
in a single chunk (exercising downloading of objects lacking a length
response header).
r  )r   r   r  zContent-Encoding:gzipzContent-Type:image/x-iconr   r  Nr   )rs  r   rL  r   r   gzipr   rK   rA   CalculateHashesFromContentsr  r   r   )	r   r   r,  input_filenamer^  	hash_dictin_file_md5r  out_file_md5s	            r   !test_cp_download_transfer_encoded(TestCp.test_cp_download_transfer_encoded?  s3    ""$J""je"LJ))+NNN%t-H$^Z  
>	"b&(#i00?e$++-k 
#   "FNND$z*DL9:	fd	r&(#i00?u%,,.l 
 	[,/ 
#	" 
	s   16E16E
E
E!c           	         U R                  5       nSU R                  -  nU R                  USUS9nU R                  5       nU R                  [        R
                  " [        SS5      5      S9nSS[        [        5      4S	/n[        U5         U R                  S
SU[        U5      U/SSS9nU R                  SU5        U R                  S
[        U5      U/SS9nU R                  SU5        U R                  SU5        [        US5       nU R                  UR                  5       US5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)z=Tests that resumble downloads work with check_hashes = never.r  r  r  Fr  rS  r
  r9  r  r   r  r   Tr   r  rV  r  z-Found no hashes to validate object downloadedr   r<  N)rs  r  r   r   r  r  r(   r   r[   r4   r   r   r   r   r   r   )	r   r   r   r,  r   r  r<  r   r   s	            r   -test_cp_resumable_download_check_hashes_never4TestCp.test_cp_resumable_download_check_hashes_neverc  sh    ""$J'H""j/4,4 # 6J !E,,8BC - E &'<c'lK?A	2	3~~
$&8
z
E /0,0  2f mm3V<~~tT*%5u=,0  2f
mm'0
mmCVLt8-KL  
4	3  
4	3s%   A>E"D>-E>
E	E
Ec           
         U R                  5       nU R                  S[        -  S9nSS[        [        5      4nU R                  [        R
                  " [        SU5      5      S9n[        U/5         U R                  SSXB[        U5      /SS	S
9nSSS5        U R                  SW5        U R                  SU5        g! , (       d  f       N3= f)zFTests that a not found exception is raised if bucket no longer exists.s   aarS  r
  r9  r  r   r  Tr   rG  NzDeleting bucketzbucket does not exist)rs  r   r[   r   r  r  r  r4   r   r   r   rl  s         r   'test_cp_resumable_upload_bucket_deleted.TestCp.test_cp_resumable_upload_bucket_deleted  s     ""$JG);<E$&;S\J,,fll5aD7F, G 
34	5~~
$&8
z
 -1./  1f 
6 	MM#V,MM)62 
6	5s   2B>>
Cc                    U R                  5       nU R                  USS[        -  S9nU R                  5       nSS[	        [        5      4SSS[	        [        5      4S/n[        U5         U R                  S	[        U5      U/5        [        [        U5      U R                  5      nU H1  nU R                  [        R                  R                  U5      5        M3     [        US
5       nU R!                  UR#                  5       S[        -  S5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)z<Tests that sliced object download works in the general case.r  r  r  r
  r9  )r
  test_assume_fast_crcmodr  r  r  r   r   r  N)rs  r   r[   r   r   r4   r   r   r?   r   rM  r  r   r   r  r   r   r   )r   r   r,  r   r<  tracker_filenamesr  r   s           r   test_cp_sliced_downloadTestCp.test_cp_sliced_download  s"    ""$J""j/4,2W,< # >J !E
 
(#g,75	5s7|D@	 
2	3
nndD,e45 <
u
%t}}6/
(89: 0 t6G#35KL  
4	3  
4	3s%   *B D5*)D$D5$
D2	.D55
Ec           
         U R                  5       nU R                  USSU R                  -  S9nU R                  5       nU R                  [        R
                  " [        SS5      5      S9nSS[        U R                  S-  5      4SS	[        U R                  5      4S
/n[        U5         U R                  SSU[        U5      [        U5      /SSS9nU R                  SU5        U R                  [        R                  R                  US-   5      5        [!        [#        U5      U R$                  5      nU H1  nU R'                  [        R                  R                  U5      5        M3     SSS5        [        U5         U R                  S[        U5      [        U5      /SS9nU R)                  SU5        U R'                  [        R                  R                  US-   5      5        [+        US5       n	U R-                  U	R/                  5       SU R                  -  S5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       N(= f! , (       d  f       g= f)z:Tests sliced download works when resumability is disabled.r  r  r  Fr  rS  r
  r9  r  r
  r	  4r   r  Tr   rG  not downloaded successfullyr  NrV  r  r   r  )rs  r   r  r   r  r  r(   r   r4   r   r   r   r   r   r   r  r?   r   rM  r  r  r   r   r   
r   r   r,  r   r  r<  r   r_  r  r   s
             r   #test_cp_unresumable_sliced_download*TestCp.test_cp_unresumable_sliced_download  s$   
 ""$J""j/4,3dnn,D # FJ !E,,8BC - E 
(#dnnq.@*AB	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16:
oobggnnUY%678 <
u
%t}}6/
(89: 0 
4& 
2	3~~j!4;
/t  Ef
*F3
rww~~ei&789t7T^^#;/	1  
4	3' 
4	32  
4	3s2   +CH5 A5I5/I$I5
I
I	I
I%c           
      V   U R                  5       nU R                  USSU R                  -  S9nU R                  5       nU R                  [        R
                  " [        SS5      5      S9nSS[        U R                  5      4SS	[        U R                  5      4S
/n[        U5         U R                  SSU[        U5      [        U5      /SSS9nU R                  SU5        [        [        U5      U R                  5      nU H1  nU R                  [         R"                  R%                  U5      5        M3     U R                  S[        U5      U/SS9nU R                  SU5        [        [        U5      U R                  5      nU H1  nU R'                  [         R"                  R%                  U5      5        M3     [)        US5       n	U R+                  U	R-                  5       SU R                  -  S5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)z/Tests that sliced object download is resumable.r  r  r  Fr  rS  r
  r9  r  r  r   r  Tr   rG  re  rV  r  r   r  N)rs  r   r  r   r  r  r(   r   r4   r   r   r   r?   r   rM  r   r   r   r  r  r   r   r   rf  s
             r   test_cp_sliced_download_resume%TestCp.test_cp_sliced_download_resume  s   
 ""$J""j/4,2T^^,C # EJ !E,,8BC - E 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16: <
u
%t}}6/
'789 0 ~~tT*%5u=,0  2f
mm'0 <
u
%t}}6/
(89: 0 t6DNN#:/	1 5 
4	34 5 
4	3s%   (D!H	/H	8H	
H	H
H(c           
      x   U R                  5       nU R                  USSU R                  -  S9nU R                  5       nU R                  [        R
                  " [        S5      5      S9nSS[        U R                  5      4SS[        U R                  5      4S	/n[        U5         U R                  S
SU[        U5      [        U5      /SSS9nU R                  SU5        [        [        U5      U R                  5      nU H1  nU R                  [         R"                  R%                  U5      5        M3     U R                  S
[        U5      U/SS9nU R                  SU5        U R                  SU5        [        [        U5      U R                  5      nU H1  nU R'                  [         R"                  R%                  U5      5        M3     [)        US5       n	U R+                  U	R-                  5       SU R                  -  S5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)zDTest sliced download resumability when some components are finished.r  r  r  r  rS  r
  r9  r  r  r   r  Tr   rG  re  rV  r  r  r   r  N)rs  r   r  r   r  r  r)   r   r4   r   r   r   r?   r   rM  r   r   r   r  r  r   r   r   rf  s
             r   &test_cp_sliced_download_partial_resume-TestCp.test_cp_sliced_download_partial_resume  s   
 ""$J""j/4,2T^^,C # EJ !E,,A!DE - G 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16: <
u
%t}}6/
'789 0 ~~tT*%5u=,0  2f
mm'0
mm/8 <
u
%t}}6/
(89: 0 t6DNN#:/	1 7 
4	36 7 
4	3s%   'D3H+/H	H+
H(	$H++
H9c           
         U R                  5       nU R                  USSU R                  -  S9nU R                  SS9nU R                  [        R
                  " [        SS5      5      S9nSS	[        U R                  5      4SS
[        U R                  5      4S/n[        U5         U R                  SSU[        U5      [        U5      /SSS9nU R                  SU5        U R                  [        R                  R                  US-   5      5        [!        [#        U5      U R$                  5      nU H1  nU R                  [        R                  R                  U5      5        M3     ['        US-   S5       n	U	R)                  S5        SSS5        U R                  S[        U5      U/SSS9nU R                  SU5        U R                  SU5        U R                  SU5        [!        [#        U5      U R$                  5      nU H1  nU R+                  [        R                  R                  U5      5        M3     U R+                  [        R                  R                  US-   5      5        U R+                  [        R                  R                  U5      5        SSS5        g! , (       d  f       GN = f! , (       d  f       g= f)z?Tests differing file contents are detected by sliced downloads.r  r  r  r   rS  Fr  r
  r9  r  r  r   r  Tr   rG  re  r  zr+bs   altered file contentsNr  r  zHashMismatchException: crc32c)rs  r   r  r   r  r  r(   r   r4   r   r   r   r   r   r   r  r?   r   rM  r   r   r  rf  s
             r   .test_cp_sliced_download_resume_content_differs5TestCp.test_cp_sliced_download_resume_content_differsD  sy   
 ""$J""j/4,2T^^,C # EJ -E,,8BC - E 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16: oobggnnUY%678 <
u
%t}}6/
'789 0 	!5)Q	() * ~~tT*%5u=,0./  1f mm'0
mm:FC
mm3V< <
u
%t}}6/
(89: 0 rww~~ei&789
rww~~e,-M 
4	3& *)' 
4	3s&   'CJ58J#
DJ5#
J2	-J55
Kc           
         U R                  5       nU R                  USSU R                  -  S9nU R                  5       nU R                  [        R
                  " [        SS5      5      S9nSS[        U R                  5      4SS	[        U R                  5      4SS
[        U R                  S-  5      4S/n[        U5         U R                  SSU[        U5      [        U5      /SSS9nU R                  SU5        SSS5        SS[        U R                  5      4SS	[        U R                  5      4SS
[        U R                  S-  5      4S/n[        U5         U R                  S[        U5      U/SS9nU R                  SU5        U R                  SU5        U R                  SU5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)zTests sliced download doesn't break when the boto config changes.

If the number of components used changes cross-process, the download should
be restarted.
r  r  r  Fr  rS  r
  r9  r  %sliced_object_download_component_sizerg  rc  r   r  Tr   rG  re  Nrg   )r
  r	  r  rV  z+Sliced download tracker file doesn't match r
  r  )rs  r   r  r   r  r  r(   r   r4   r   r   r   r  r  s          r   .test_cp_sliced_download_component_size_changed5TestCp.test_cp_sliced_download_component_size_changed  s    ""$J""j/4,3dnn,D # FJ !E,,8BC - E 
(#dnn*=>	5s4>>7JK	:	T^^q 	!	#@ 
2	3~~
$&8
z

u+
 -1./  1f mm16: 
4 
(#dnn*=>	5s4>>7JK	:	T^^q 	!	#@ 
2	3~~tT*%5u=,0  2f
mmBFK
mm6?
*F3 
4	3% 
4	3$ 
4	3s   ;F7AG7
G
Gc           
         U R                  5       nU R                  USSU R                  -  S9nU R                  5       nU R                  [        R
                  " [        SS5      5      S9nSS[        U R                  5      4SS	[        U R                  5      4S
/n[        U5         U R                  SSU[        U5      [        U5      /SSS9nU R                  SU5        U R                  [        R                  R                  US-   5      5        [!        [#        U5      U R$                  5      nU H1  nU R                  [        R                  R                  U5      5        M3     SSS5        SS[        U R                  5      4SS	[        U R                  S-  5      4S
/n[        U5         U R                  S[        U5      U/SS9nU R'                  SU5        U R)                  [        R                  R                  US-   5      5        W H1  nU R)                  [        R                  R                  U5      5        M3     [+        US5       n	U R-                  U	R/                  5       SU R                  -  5        SSS5        SSS5        g! , (       d  f       GN0= f! , (       d  f       N)= f! , (       d  f       g= f)zTests temporary files are not orphaned if sliced download is disabled.

Specifically, temporary files should be deleted when the corresponding
non-sliced download is completed.
r  r  r  Fr  rS  r
  r9  r  rc  r   r  Tr   rG  re  r  NrV  r  r   )rs  r   r  r   r  r  r(   r   r4   r   r   r   r   r   r   r  r?   r   rM  r  r  r   r   r   rf  s
             r   .test_cp_sliced_download_disabled_cross_process5TestCp.test_cp_sliced_download_disabled_cross_process  sw    ""$J""j/4,3dnn,D # FJ !E,,8BC - E 
(#dnn*=>	5s4>>7JK@ 
2	3~~
$&8
z

u+
 -1./  1f mm16:
oobggnnUY%678 <
u
%t}}6/
'789 0 
4( 
(#dnn*=>	5s4>>A;M7NO@ 
2	3~~tT*%5u=,0  2f
*F3
rww~~ei&789 0
(89: 0t7T^^#;<  
4	33 
4	3F  
4	3s2   (CJ1B#J6.J%J6
J"%
J3	/J66
Kc                 8    S H  nU R                  U5        M     g )N)i    i  )start_over_error_test_helper)r   start_over_errors     r   .test_cp_resumable_upload_start_over_http_error5TestCp.test_cp_resumable_upload_start_over_http_error  s      ''(89	r   c           
      ^   U R                  5       n[        [        S-  S9nU R                  US9nSS[	        [
        5      4nU R                  [        R                  :X  a.  U R                  [        R                  " [        SS5      5      S9nOJU R                  [        R                  :X  a,  U R                  [        R                  " [        S5      5      S9n[        U/5         U R                  SS	WU[!        U5      /S
S9nU R#                  SU5        S S S 5        g ! , (       d  f       g = f)Nrg  r  rS  r
  r9  r  rz  r   r  TrV  zRestarting upload of)rs  rW   r\   r   r   r[   rM  r   JSONr  r  r   rN  r  r4   r   r   r   )r   r   r   
rand_charsr   r<  r  r   s           r   r{  #TestCp.start_over_error_test_helper  s   ""$J (gk;J4E$&;S\J}}(((..<< Fq# NO / Q	+//	)..
9!
<9>. ? 
34	5~~
$&8%
z
 -1	  2f
 mm*F3 
6	5	5s   $1D
D,c                     U R                  5       nU R                  USSS9nSS[        U5      S-   [        U5      [        U5      S-   /nU R                  USS	9  U R                  S
S[        U5      -  /5        g )Nr  r   r  r   z-cz/foo2z/dir/r   )r   r   z
%s/dir/foo)rs  r   r   r   )r   r   r,  
cp_commands       r   test_cp_minus_cTestCp.test_cp_minus_c  s    ""$J""j/4,2 # 4J
 	Z7"ZZ7"J 	NN:qN1NNFL4
+;;<=r   c           	      .   U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       nU R                  USSS9n[        [        [        R                  " 5       [        5       U R                  5      nUR                  5       n[        R                  " UR                   UR"                  R                   UR$                  S9n[        R                  " UR"                  U R'                  S5      UR(                  S9nUR+                  XV5        U R-                  UR/                  UR"                  UR                   SS	/S
9R0                  UR/                  UR"                  UR                   SS	/S
9R0                  S5        g)zTests the JSON Rewrite API.&Rewrite API is only supported in JSON.r  rT  r  )r(  bucketcontentTypeobjectr  r(  r  customerEncryptionmd5Hashfields;Error: Rewritten object's hash doesn't match source object.N)rM  r   rN  r;   r  rs  r   r   r   logging	getLoggerr   r   rK  apitools_messagesObjectr(  r  r  MakeTempNamer  
CopyObjectr   GetObjectMetadatar  )r   r   r,  
gsutil_apikeysrc_obj_metadatadst_obj_metadatas          r   test_rewrite_cpTestCp.test_rewrite_cp  sv   }}']]CDD""$J""j/4,2 # 4J ,g.?.?.A02D4I4IKJ



C(//SXX7:zz<?<L<LN )//&&x($002 *=$$%5%<%<%5%:%:-A-6-8 	% 	9 :A$$%5%<%<%5%:%:-A-6-8 	% 	9 :AG	Ir   c           	         U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       nU R                  SS9nU R                  USS[        -  S-   SS9n[        [        [        R                  " 5       [        5       U R                  5      nUR                  5       n[        R                   " UR"                  UR$                  R"                  UR&                  UR(                  R+                  S	5      S
9nU R-                  S5      n[        R                   " UR.                  UUR0                  S9n[3        UR$                  UR"                  UR$                  UR"                  U R                   5      n	  UR5                  UU[7        [        S-  5      R8                  [        S9  U R;                  S5        U R?                  [@        RB                  RE                  U	5      5        UR5                  UU[G        [        S-  5      R8                  [        S9  U RI                  [@        RB                  RE                  U	5      5        U RK                  URM                  UR$                  UR"                  SS/S9RN                  URM                  UR$                  UR"                  SS/S9RN                  S5        [Q        U	5        g! [<         a     GNf = f! [Q        U	5        f = f)zETests the JSON Rewrite API, breaking and resuming via a tracker file.r  r  r  r     12rT  Tr~  "'r(  r  r  r  r  r  rg   progress_callbackmax_bytes_per_callExpected RewriteHaltException.r  r  r  r  N))rM  r   rN  r;   r  rs  r   r\   r   r   r  r  r   r   rK  r  r  r(  r  r  r  r  r  r   r  r>   r  r   r  r  r   r   r   r   r  r   r  r   r  r  r=   )
r   r   bucket_uri2r,  r  r  r  dst_obj_namer  r  s
             r   test_rewrite_cp_resumeTestCp.test_rewrite_cp_resume:  s   }}']]CDD""$J ##4 $ 6K
 ""j/4-2W_,F37 # 9J ,g.?.?.A02D4I4IKJ



C(//SXX7:zz<?<L<L58XX^^E5JL $$X.L(//&&$002 22B2I2I2B2G2G2B2I2I2B2G2G26--	A
%+..0M")A+1//3t18	 	 	:
 			23
 oobggnn%678 

>w?@@A BBF$$  & rww~~&789


&
&'7'>'>'7'<'</C/8/: ' ; <C7

&
&'7'>'>'7'<'</C/8/: ' ; <C7
I	K )*; " : )*s,   AK C=K( 
K%!K( $K%%K( (K5c           	         U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       nU R                  SS9nU R                  USS[        -  S-   SS9n[        [        [        R                  " 5       [        5       U R                  5      nUR                  5       n[        R                   " UR"                  UR$                  R"                  UR&                  UR(                  R+                  S	5      S
9nU R-                  S5      n[        R                   " UR.                  UUR0                  S9n[3        UR$                  UR"                  UR$                  UR"                  U R                   5      n	  UR5                  UU[7        [        S-  5      R8                  [        S9  U R;                  S5        U R                  USSSS9n
U
R                  5       n[        R                   " UR"                  UR$                  R"                  UR&                  UR(                  R+                  S	5      S
9nU R?                  [@        RB                  RE                  U	5      5        UR5                  UU[        S9  U RG                  [@        RB                  RE                  U	5      5        U RI                  URK                  UR$                  UR"                  SS/S9RL                  URK                  UR$                  UR"                  SS/S9RL                  S5        [O        U	5        g! [<         a     GNuf = f! [O        U	5        f = f)zBTests that Rewrite starts over when the source object has changed.r  r  r  r  r  rT  Tr~  r  r  r  r  rg   r  r  )r  r  r  r  r  N)(rM  r   rN  r;   r  rs  r   r\   r   r   r  r  r   r   rK  r  r  r(  r  r  r  r  r  r   r  r>   r  r   r  r  r   r   r   r   r  r  r   r  r  r=   )r   r   r  r,  r  r  r  r  r  r  object_uri2key2src_obj_metadata2s                r   %test_rewrite_cp_resume_source_changed,TestCp.test_rewrite_cp_resume_source_changed  s   }}']]CDD""$J ##4 $ 6K
 ""j/4-2W_,F37 # 9J ,g.?.?.A02D4I4IKJ



C(//SXX7:zz<?<L<L58XX^^E5JL $$X.L(//&&$002 22B2I2I2B2G2G2B2I2I2B2G2G26--	A
-+..0M")A+1//3t18	 	 	:
 			23 %%27/56: & <k   "d+22yy!!''yyu%	' oobggnn%678 -,/6  8
 rww~~&789


&
&'8'?'?'8'='=/C/8/: ' ; <C7

&
&'7'>'>'7'<'</C/8/: ' ; <C7
I	K )*K " J )*s,   AL6 EM 6
M M MM Mc           	      8   U R                   [        R                  :X  a  [        R                  " S5      $ U R                  5       nU R                  SS9nU R                  USS[        -  S-   SS9n[        [        [        R                  " 5       [        5       U R                  5      nUR                  5       n[        R                   " UR"                  UR$                  R"                  UR&                  UR(                  R+                  S	5      S
9nU R-                  S5      n[        R                   " UR.                  UUR0                  S9n[3        UR$                  UR"                  UR$                  UR"                  U R                   5      n	  UR5                  UUS[7        [        S-  5      R8                  [        S9  U R;                  S5        U R?                  [@        RB                  RE                  U	5      5        UR5                  UUS[        S9  U RG                  [@        RB                  RE                  U	5      5        URI                  UR$                  UR"                  / SQS9n
U RK                  URI                  UR$                  UR"                  SS/S9RL                  U
RL                  S5        SnU
RN                   H  nURP                  S:X  d  M  SnM     U R?                  US5        [S        U	5        g! [<         a     GN9f = f! [S        U	5        f = f)z:Tests that Rewrite starts over when the arguments changed.r  r  r  r  r  rT  Tr~  r  r  r  r  privaterg   )
canned_aclr  r  r  r  )r  r  )r  r  r  r  r  r  r  FallUsersz-New object was not written with a public ACL.N)*rM  r   rN  r;   r  rs  r   r\   r   r   r  r  r   r   rK  r  r  r(  r  r  r  r  r  r   r  r>   r  r   r  r  r   r   r   r   r  r  r  r   r  r  entityr=   )r   r   r  r,  r  r  r  r  r  r  new_obj_metadatafound_public_acl	acl_entrys                r   &test_rewrite_cp_resume_command_changed-TestCp.test_rewrite_cp_resume_command_changed  s   }}']]CDD""$J ##4 $ 6K
 ""j/4-2W_,F37 # 9J ,g.?.?.A02D4I4IKJ



C(//SXX7:zz<?<L<L58XX^^E5JL $$X.L(//&&$002 22B2I2I2B2G2G2B2I2I2B2G2G26--	A
,+	..)20M")A+1//3t18 	 	: 			23
 oobggnn%678 ,,'4/6  8 rww~~&789#55

!
!


9 6 ; 

&
&'7'>'>'7'<'</C/8/: ' ; <C7

"
"
IK '++)z)!
 , oo&EG )*G " F )*s2   AK; DL L ;
L	L L		L L*POSIX attributes not available on Windows.zTest requires fast crcmod.c                 Z    U R                  5       nU R                  5       n[        XUSS9  g)zTests use of the -P flag with cp from a bucket to a local dir.

Specifically tests combinations of POSIX attributes in metadata that will
pass validation.
Tr   N)rs  r  r   )r   r   r   s      r   .test_cp_preserve_posix_bucket_to_dir_no_errors5TestCp.test_cp_preserve_posix_bucket_to_dir_no_errors!  s,     ""$J!F&ttLr   c                 |    U R                  5       nU R                  5       nU R                  USSS9n[        XX2SS9  g)zTests use of the -P flag with cp from a bucket to a local dir.

Specifically, combinations of POSIX attributes in metadata that will fail
validation.
r   s   objr  Tr  N)rs  r  r   r   )r   r   r   r   s       r   +test_cp_preserve_posix_bucket_to_dir_errors2TestCp.test_cp_preserve_posix_bucket_to_dir_errors-  sK     ""$J!F


z(-%+  -C %Ts$Or   c                 8    U R                  5       n[        XSS9  g)z>Tests use of the -P flag with cp from a local dir to a bucket.Tr  N)rs  r   )r   r   s     r   -test_cp_preseve_posix_dir_to_bucket_no_errors4TestCp.test_cp_preseve_posix_dir_to_bucket_no_errors<  s     ""$J&ttDr   c                     U R                  SS9nU R                  5       nU R                  SSSX/SSS9nU R                  S	U5        g
)zDTest that cp -s operations to a non-cloud destination are prevented.r   rS  r   r  r  r   Tr   z9Cannot specify storage class for a non-cloud destination:N)r   r  r   r   )r   
local_filedest_dirr   s       r   'test_cp_minus_s_to_non_cloud_dest_fails.TestCp.test_cp_minus_s_to_non_cloud_dest_failsB  s[    $$f$5J!!#H^^T4ZJ,-*.  0F 	MMMr   zNeed Boto version > 2.46.1c                 .   U R                  5       nU R                  USSS9n[        U5      S-   nSSS.nX@R                     nU R	                  SS	U[        U5      U/5        U R	                  S
U/SS9nU R                  USU-  [        R                  S9  g )Nr  r   r  r  Standard_iAdurable_REDUCED_availability)r  r  r   r  r   Tr  zStorage class:\s+%sr  )rs  r   r   r   r   r  r   r  )r   r   r,  object2_surinondefault_storage_classr  r  s          r   (test_cp_specify_nondefault_storage_class/TestCp.test_cp_specify_nondefault_storage_classN  s    ""$J""j/4,2 # 4J 
#e+L
 ,  --B-BCMNND$tJ/?NO^^V\2$^GF%%f&<}&L,.MM & ;r   z&Test uses gs-specific storage classes.c                 $   U R                  SS9nSnU R                  SUS9n[        U5      S-   U-   nU R                  SSS	X4/5        [	        S
/5         U R                  SU/SS9nSSS5        U R                  WS[        R                  S9  [        U5      S-   nU R                  SXF/5        [	        S
/5         U R                  SU/SS9nSSS5        U R                  US[        R                  S9  [        U5      S-   nU R                  SSS	Xg/5        [	        S
/5         U R                  SU/SS9nSSS5        U R                  US[        R                  S9  g! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       NP= f)zETests that object storage class is set correctly with and without -s.r  r  zfoo-origr   rV  r   r   r  r  r  r   Tr  NzStorage class:\s+STANDARDr  z/foo-nlzStorage class:\s+NEARLINEz/foo-std)rs  r   r   r   r4   r  r   r  )r   r   local_fnamelocal_fpathfoo_cloud_surir  foo_nl_surifoo_std_suris           r   'test_cp_sets_correct_dest_storage_class.TestCp.test_cp_sets_correct_dest_storage_classc  s    """<JK%%v%MK*%+k9NNND$
KHI	?@	A~~v~6d~Kf 
B%%f&B,.MM & ; z"Y.KNND.67	?@	A~~v{34~Hf 
B%%f&B,.MM & ; 
#j0LNND$
KFG	?@	A~~v|4D~If 
B%%f&B,.MM & ;1 
B	A 
B	A 
B	As$   E;E0$F
E-0
E>
Fc                    U R                  5       nSnSn[        5       nU R                  UUSS9n[        SSU4/5         U R	                  S[        U5      [        U5      < SU< 3/5        S S S 5        [        S	/5         U R                  [        U5      < SU< 3U5        S S S 5        g ! , (       d  f       NF= f! , (       d  f       g = f)
Nr  r  r   r  r
  rJ  r   r   r  )rs  r%   r   r4   r   r   r  )r   r   	obj1_name	obj2_namer  r  s         r   >test_kms_key_correctly_applied_to_dst_obj_from_src_with_no_keyETestCp.test_kms_key_correctly_applied_to_dst_obj_from_src_with_no_key  s    ""$JII02G :,5)/   1G 
*:GDE	F
nng:&	245 
G 
?@	A
4
+;Y G ') 
B	A 
G	F 
B	As   .B-#B>-
B;>
Cc                 b   U R                  5       nU R                  SS9nSn[        U5      S-   U-   n[        5       n[	        SSU4/5         U R                  SX$/5        S S S 5        [	        S/5         U R                  XE5        S S S 5        g ! , (       d  f       N5= f! , (       d  f       g = f)	Nr  rS  r  r   r
  rJ  r   r  r  )r   r   r   r   r  r  s         r   9test_kms_key_correctly_applied_to_dst_obj_from_local_file@TestCp.test_kms_key_correctly_applied_to_dst_obj_from_local_file  s    ""$J1EHJ#%0H02G	*:GDE	F
nndE,- 
G 
?@	A
2 
B	A 
G	F 
B	As   B4B 
B 
B.c                    SnU R                  5       nU R                  SU-  S9nSn[        U5      S-   U-   n[        5       n[	        SSU4SS[        U5      4/5         U R                  S	X5/5        S S S 5        [	        S
/5         U R                  XV5        S S S 5        g ! , (       d  f       N5= f! , (       d  f       g = f)Nr  r  rS  r  r   r
  rJ  r9  r   r  )rs  r   r   r%   r4   r   r   r  )r   r9  r   r   r   r  r  s          r   (test_kms_key_works_with_resumable_upload/TestCp.test_kms_key_works_with_resumable_upload  s    %""$J0C)CDEHJ#%0H02G	*:GD (*? #$7 8 :; 
< nndE,-
<
 
?@	A
2 
B	A
< 
<
 
B	As   B!B2!
B/2
C c           	         U R                  5       nSnSn[        5       n[        [        R                  S9n[	        U R                  UUSUS95      n[	        U5      < SU< 3n[        SSU4/5         U R                  S	Xg/5        S S S 5        [        S
/5         U R                  Xu5        S S S 5        g ! , (       d  f       N5= f! , (       d  f       g = f)Nr  r  )key_namer   r   r   r   kms_key_namer   r
  rJ  r   r  )	rs  r%   r,   CONSTANT_KEY_NAME2r   r   r4   r   r  )r   r   r  r  key1_fqnkey2_fqn	obj1_suri	obj2_suris           r   @test_kms_key_correctly_applied_to_dst_obj_from_src_with_diff_keyGTestCp.test_kms_key_correctly_applied_to_dst_obj_from_src_with_diff_key  s    ""$JII13H1$779HZ&/#)'/ 	 	12I  
+Y7I	*:HEF	G
nndI12 
H 
?@	A
	4 
B	A	 
H	G 
B	As   2B5C5
C
Cz5Copying KMS-encrypted objects prohibited with XML APIc           	      .   U R                  5       nSnSn[        5       n[        U R                  UUSUS95      n[        U5      < SU< 3nU R	                  SXV/5        [        S/5         U R                  U5        S S S 5        g ! , (       d  f       g = f)Nr  r  r   r  r   r   r  )rs  r%   r   r   r   r4   AssertObjectUnencrypted)r   r   r  r  r  r  r  s          r   @test_kms_key_not_applied_to_nonkms_dst_obj_from_src_with_kms_keyGTestCp.test_kms_key_not_applied_to_nonkms_dst_obj_from_src_with_kms_key  s     ""$JII13HZ&/#)'/ 	 	12I  
+Y7INND)/0 
?@	A
""9- 
B	A	As   +B
BzWOnly Windows paths need to be normalized to use backslashes instead of forward slashes.c                     U R                  5       nU R                  USSS9  U R                  5       nU R                  SSU-  [	        U5      /5        U R                  SS[	        U5      -  /5        g )Nr   r   )r   rD  r   r   z	%s\./obj1r   z%s/obj1)r  r   rs  r   r   )r   r@  r   s      r   ;test_windows_path_with_back_and_forward_slash_is_normalizedBTestCp.test_windows_path_with_back_and_forward_slash_is_normalized  so       "Gw&6J""$JNND,0$z2BCD 	NNFIZ(889:r   c                     U R                  / SQSSS9nU R                  (       a  U R                  SU5        gU R                  SU5        g)z*Tests that cp -m - anything is disallowed.)rX  r   r  r  Tr   rG  zSWARNING: Using sequential instead of parallel task execution to transfer from stdinzBCommandException: Cannot upload from a stream when using gsutil -mNrh  )r   r   s     r    test_cp_minus_m_streaming_upload'TestCp.test_cp_minus_m_streaming_upload  sQ    ^^5*.,-  /F 
mm!"(* mm
N
r   c                    U R                  SS9nU R                  SS9nU R                  S[        U5      U/SS9n[	        US5       nU R                  UR                  5       S5        S S S 5        g ! , (       d  f       g = f)Nr   rS  rT  r   TrV  r   )r   r   r   r   r   r   r   r]  s        r   'test_cp_overwrites_existing_destination.TestCp.test_cp_overwrites_existing_destination	  ss    0G0E^^T4=%8^MF	eT	a
qvvx( 
		s   !A11
A?c                    SnU R                  5       nU R                  US9nS[        US5      U/n/ n[        S5       HD  n[        R
                  " U R                  U/S9nUR                  5         UR                  U5        MF     U Vs/ s H  oR                  5       PM       nU R                  [        [        R                  " U5      5      U5        g s  snf )Nr  r  r   rb  rg   r  )r  rs  r   r_   r  r  r   r  r   r   r   rk  r   listdir)	r   test_file_counttemporary_directoryr   cp_argsthreadsrm  threadts	            r   >test_downloads_are_reliable_with_more_than_one_gsutil_instanceETestCp.test_downloads_are_reliable_with_more_than_one_gsutil_instance  s    O,,."""@JT*c*,?@GG1Xt~~WIFfllnnnV  w!VVXwS$789?K  s   
C)rJ  )N)r  r  r  r  r	  rC   r  rG  rP  r3   r_  rc  rn  ry  r#   r  r  r;   skipIfrV   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r$  r,  r4  r=  r$   r@  rT  r[  r"   rw  r~  r  r  r  r  r  r  
skipUnlessr+   r  r  r  r  r  r  r*   r  r  r  r  r   r	  r  r  r  r)  r9  r>  r!   rK  r]  rb  rh  rk  ro  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"  r'  r.  r5  r<  rA  rE  rP  rS  rX  r_  rc  rf  ri  rm  rs  r{  r  r  r  r  r  r  r  r  r  r  r  r  r    r  r  r  r  r  r   r  rB   r  r  r   r$  r)  r/  r2  rG  rL  rU  rX  r[  r`  rg  rj  rm  rp  rt  rw  r}  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r   r   r@  r@  c  s   ) '*)BE !) !) !* !*"E  !EF/ G !/: !EF+ G !+> !EF??:JK"O L G !"OH ??:DE ; ! F;& ??:DE ; ! F;( ??:DE 1 ! F1. !? !? ??:DE %H ! F%HN ??:DE K ! FK& ??:DE ; ! F;.  ! !<:> ??:GH  ! I$ !( !(T ! !> !7 !7. !+ !+& !:;M < !M* !<=G > !G$ !/G !/Gb& EFN GN` !235 4 !54 !23(  4 !( T !/ !/  !EF'< G !'<R& ! !&= |%OP: Q: |%OP== ' (<( Q< |%OP Q == < =?=?*$L 
o LMNMN0"1$1$. !@ !@@41 !, !,&8.6 BC# D#J!F |%OP! Q!F  < ="="8"8	2: ??:OQ B !QB
 !: !:2.h !-' !-'^ !> !>8 !- !-< ?@IJ ^, ! K A^,@ ?@IJ  ! K A2 ?@IJ  ! K A2 !- !-@ ?@IJ -- ! K A--^6.606./	/ !8 !8"3, ??:JKA LA  ??:JK$N L$NL< !	2 !	2 LM 2 ! N2< LM 1 ! N1& LM 2 ! N22 @AD BD8 @A!D B!DF @A!F B!FF/
 23/ 4/( ?@IJ &+ ! K A&+P 23* 4*8 23( 4(, 67P 8P 23? 4?& 236 46 23, 4,: 23= 4=< 23"D 4"DH 23 = 4 =D 23@+ 4@+D 233+ 43+j 343 53  23= 4=" 23??:EF $ ! G 4 $2 ??:EF $ ! G $: ?C$HL LM- N- LM0H N0Hd !4 !4F +->@12%J 3@%JN56-$^"$H$$L	; ?@IJ ; ! K A;$ ??:FG I= ! HI=V'"0H !M !M8 233 43$ 12M 3M: +->@12+1 3@+1Z +->@12,1 3@,1\ +->@12-1 3@-1^ +->@126. 3@6.p +->@12-4 3@-4^ +->@128= 3@8=t 23: 4:4,> I<F+PN+`M+^ ??:KL+-/KLM M MM ??:KLP MP ??:KLE ME
 *+; ,;( 56#; 7#;J 34) 5)* 343 53 343 53  345 55, 34EF. G 5.& ;	; !) !) !L !Lr   r@  c            	       `   \ rS rSrSrS rS rS rS r\	R                  " \S5      \R                  " S\R                  " S	S
9S9\R                  R                  \SSS9S 5       5       5       r\	R                  " \S5      \R                  " S\R                  " SS
9S9\R                  R                  \SSS9S 5       5       5       r\R                  " S5      \R                  " S5      \R                  " S\R                  " \R(                  R*                  S9S9S 5       5       5       rS rSrg)TestCpUnitTestsi"  zUnit tests for gsutil cp.c                 l   U R                  SS9nU R                  USS9nSUR                  5       l        U R	                  5       nU R                  S[        U5      U/SS9nUR                  S	   nU R                  S
[        U5      5        U R                  US   S5        U R                  SUS   5        g)4Tests a download with no valid server-supplied hash.r  r  r   rq  12345r   Treturn_log_handlerwarningrg   r   zQNon-MD5 etag \(12345\) present for key .*, data integrity checks are not possiblezIntegrity cannot be assuredr   N)rs  r   rK  r  r  
RunCommandr   messagesr   rk  rY  r   )r   r   r,  rX  log_handlerwarning_messagess         r   testDownloadWithNoHashAvailable/TestCpUnitTests.testDownloadWithNoHashAvailable%  s     ""D"1J""j6"JJ 'J  "G//$j)97(C59 " ;K"++I6Q,-. 23 	MM/1A!1DEr   c                 J   U R                  SS9nU R                  USS9nS[        R                  R                  -   nU R                  [        5       nU R                  S[        U5      U/5        U R                  [        U5      S5        SSS5        g! , (       d  f       g= f)	r  r  r  r   rq  
random_dirr   z%Invalid destination path: random_dir/N)rs  r   r   r   r:  assertRaisesr   r  r   r   r   )r   r   r,  destination_patherrors        r   9testDownloadWithDestinationEndingWithDelimiterRaisesErrorITestCpUnitTests.testDownloadWithDestinationEndingWithDelimiterRaisesError6  s     ""D"1J""j6"JJ#bggkk1			?	+u
oodT*-/?@A
s5z#JK 
,	+	+s   9B
B"c                    U R                  5       nU R                  USSS9nU R                  USSS9  U R                  5       n[        S/5         U R	                  S[        U5      U/5        S S S 5        [        US5       nU R                  UR                  5       S5        S S S 5        g ! , (       d  f       ND= f! , (       d  f       g = f)	Nr  r   r  zfoo/barrT  r  r   r   )	rs  r   r   r4   r  r   r   r   r   )r   r   r,  r   r   s        r    test_object_and_prefix_same_name0TestCpUnitTests.test_object_and_prefix_same_nameA  s    ""$J""j/4,2 # 4J 	"+%  ' !E	BC	D
oodT*-u56 
E	eT	a
qvvx( 
	 
E	D		s   B+!B<+
B9<
C
c                 D   U R                  5       nU R                  SS9n[        S/5         U R                  SU[	        U5      /SS9nS S S 5        WR
                  S   nU R                  S[        U5      5        U R                  S	US
   5        g ! , (       d  f       NN= f)Nr  rS  r  r   Tr  r  r   z)Found no hashes to validate object uploadr   )	rs  r   r4   r  r   r  r   rk  r   )r   r   r   r  r  s        r   !test_cp_upload_respects_no_hashes1TestCpUnitTests.test_cp_upload_respects_no_hashesP  s    ""$J1E	BC	DOOD5$z2B*C7; $ =k 
E #++I6Q,-.MM="1%' 
E	Ds   B
Br  z
os.geteuidr   )return_value)newchownT)autospecc                     U R                  SS9n[        R                  " 5       n[        [        [
        0S9Ul        [        X#SS5        UR                  U[
        S5        g )Nr  rS  entriesFT)	r   r  r  rL   rQ   rf   metadatarR   assert_called_once_withr   
mock_chownr   r   s       r   #test_posix_runs_chown_as_super_user3TestCpUnitTests.test_posix_runs_chown_as_super_user[  sT     1E

"
"
$C'70CDCLu5$7&&ugr:r   r   c                     U R                  SS9n[        R                  " 5       n[        [        [
        0S9Ul        [        X#SS5        UR                  5         g )Nr  rS  r)  FT)	r   r  r  rL   rQ   rf   r,  rR   assert_not_calledr.  s       r   *test_posix_skips_chown_when_not_super_user:TestCpUnitTests.test_posix_skips_chown_when_not_super_usere  sN     1E

"
"
$C'70CDCLu5$7  "r   zFgslib.utils.copy_helper.TriggerReauthForDestinationProviderIfNecessaryz/gslib.command.Command._GetProcessAndThreadCountzgslib.command.Command.Apply)specc                    U R                  SS9nU R                  5       nSUl        U R                  SU[	        U5      /5        UR                  [        [	        U5      5      [        R                  S5        UR                  S S S SS9  g )Nr  r  )rg   rf  r   r  F)process_countthread_countparallel_operations_overrideprint_macos_warning)	r   rs  r$  r  r   r-  r   r   ANY)r   !mock_get_process_and_thread_countmock_trigger_reauthr   r   s        r   test_cp_triggers_reauth'TestCpUnitTests.test_cp_triggers_reautho  s     %1D""$J59%2OOD4j!123//T*-.	 &==%)!	 > r   c                 J    / SQn[        U5        U R                  U/ SQ5        g )N)z
--flag-keyz
flag-value)r  r  r  zdoes-not-exist)rB  )r  
publicReadrC  )r   r   )r   sub_optss     r   'test_translates_predefined_acl_sub_opts7TestCpUnitTests.test_translates_predefined_acl_sub_opts  s(    *H+H5X  P Qr   r  N)r  r  r  r  r	  r  r  r  r"  r;   r  rV   r   patchMockr  r   r0  r4  r   CommandApplyr?  rF  r
  r  r   r   r
  r
  "  s6   !F"	L)	' ??:KL::l		q 9:::R40; 1 ; M; ??:KL::l		q 9:::R40# 1 ; M# ::NP::?@::+))!6!6799 AP
*Qr   r
  c                       \ rS rSrSrS rSrg)TestCpShimUnitTestsi  z Unit tests for shimming cp flagsc                 H   U R                  5       nU R                  SS9n[        SS/5         [        SSS.5         U R	                  SS	S
SSSSSSSSU[        U5      /SS9nSR                  UR                  S   5      nU R                  SR                  [        R                  " S5      U[        U5      5      U5        SR                  UR                  S   5      nU R                  SU5        S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)Nr  rS  )r
  use_gcloud_storager  )r
  hidden_shim_modedry_runr  fake_dir)(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTILCLOUDSDK_ROOT_DIRr   r3  rU  r  ra  r  z
some-classr  z-Ur  r  Tr  rf  infozGcloud Storage Command: {} storage cp --ignore-symlinks --no-clobber -r -r --storage-class some-class --print-created-message --skip-unsupported --predefined-acl publicRead {} {}r  z%Use the -m flag to enable parallelism)rs  r   r4   r5   r  r   r   r  r   rZ  r]   _get_gcloud_binary_path)r   r   r   mock_log_handler
info_lines
warn_liness         r   test_shim_translates_flags.TestCpShimUnitTests.test_shim_translates_flags  s8   ""$J1E	GHJ 
K 6<)" 	  ??4$dD,dD52

 ?C + D YY/88@A
\\b\b11*=uZ ]"#-	/ YY/88CD
=zJ%	
K 
K	 	
K 
Ks#   DB5D1D
D	D
D!r  N)r  r  r  r  r	  rZ  r
  r  r   r   rM  rM    s    (Kr   rM  )T)r	  
__future__r   r   r   r   r  ry  rw  r  rO  r  r   r  rE  r=  r   r   r@  r   r  r;   r   apitools.base.pyr	   r   r  r
   boto.exceptionr   r   boto.storage_urir   rB  r   r   r   gslib.cloud_apir   gslib.commands.configr   gslib.commands.cpr   gslib.cs_api_mapr   gslib.daisy_chain_wrapperr   gslib.discard_messages_queuer   gslib.exceptionr   gslib.gcs_json_apir   gslib.parallel_tracker_filer   r   gslib.project_idr   gslib.storage_urlr   gslib.tests.rewrite_helperr   r   r   gslib.tests.testcasetestsr  gslib.tests.testcase.baser    )gslib.tests.testcase.integration_testcaser!   r"   r#   r$   gslib.tests.utilr%   r&   r'   rI  r(   r)   r*   r+   r,   r-   r   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   "gslib.third_party.storage_apitoolsr<   r  gslib.tracker_filer=   r>   r?   gslib.ui_controllerr@   gslib.utilsrA   gslib.utils.boto_utilrB   gslib.utils.constantsrC   rD   gslib.utils.copy_helperrE   rF   rG   rH   gslib.utils.hashing_helperrI   rJ   rK   gslib.utils.metadata_utilrL   gslib.utils.posix_utilrM   rN   rO   rP   rQ   rR   rS   rT   gslib.utils.retry_utilrU   gslib.utils.system_utilrV   gslib.utils.text_utilrW   gslib.utils.unit_utilrX   rY   rZ   r[   r\   r]   r   	six.movesr^   r_   r  r   r8  gslib.testsr`   ra   rb   rc   rd   re   rf   r  r   r   r   r   r   r  r   r  r  r0  r  r@  GsUtilUnitTestCaser
  ShimUnitTestBaserM  r  r   r   <module>r     s#   ( &  % ' 
      	    	   
   >   7 / -     = J E ( B = + ) 9 F . 2 I D ; ' ' 7 ? ? @ A ? , 8 7 @ ( ) 0 0 * , < - , : 1 2 $ 1 < 1 1 % W 0 8 @ 7 & 6 : & 6 ? B 3 I ? - : + , ( * + = ? 4 ( . 8 + 6 3 ) ) ! 
 !  777	$ +,/,*&-* )Y)xm.`I<X%

6V 6.Pf P,QF QB<6 <0|BLX// |BL~ElQh11 lQ^K(33 Kr   