
                            S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
r
SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJ r   SSK!J"r"  SSK#J$r$  SSK%J&r&  SSK'J(r(  SSK'J)r)  SSK'J*r*  SSK+J,r-  SSK.J/r/  SSK.J0r0  SSK.J1r1  SSK.J2r2  SSK.J3r3  SS K4J5r5  SS!K4J6r6  SS"K4J7r7  SS#K4J8r8  SS$K4J9r9  SS%K4J:r:  SS&K4J;r;  SS'K4J<r<  SS(K4J=r=  SS)K4J>r>  SS*K?J@r@JArA  \@" \A" S+S+S,5      5        SS-KBJCrC  \3R                  rE " S. S/\&5      rF " S0 S1\&5      rGg)2z8Unit tests for parallel upload functions in copy_helper.    )absolute_import)print_function)division)unicode_literalsN)
exceptions)BucketListingObject)BucketListingPrefix)CloudApi)ResumableUploadAbortException)ResumableUploadException)!ResumableUploadStartOverException)ServiceException)CreateOrGetGsutilLogger)DiscardMessagesQueue)CommandException)
GcsJsonApi)ObjectFromTracker)StorageUrlFromString)MockCloudApi)GsUtilUnitTestCase)GSMockBucketStorageUri)SetBotoConfigForTest)unittest)storage_v1_messages)copy_helper)parallelism_framework_util)
posix_util)system_util)hashing_helper)_CheckCloudHashes)_DelegateUploadFileToObject)_GetPartitionInfo) _SelectUploadCompressionStrategy)_SetContentTypeFromFile)ExpandUrlToSingleBlr)FilterExistingComponents)GZIP_ALL_FILES)%PerformParallelUploadFileToObjectArgs)"WarnIfMvEarlyDeletionChargeApplies)add_moveMovedModulemockzunittest.mock)r,   c                      \ rS rSrSrS rS rS rS rS r	S r
S	 rS
 rS rS rS rS rS rS r\R&                  " SSSSSS5      r\R,                  " S\R.                  " \R2                  " \5      S9S9S 5       rS rS rS rS r\R,                  " S5      S  5       r\R,                  " S!5      S" 5       r \R,                  " S!5      S# 5       r!S$ r"S%r#g&)'TestCpFuncsI   z7Unit tests for parallel upload functions in cp command.c                    [        SSS5      u  pU R                  SU5        U R                  SU5        [        SSS5      u  pU R                  SU5        U R                  SU5        [        SSS5      u  pU R                  SU5        U R                  SU5        [        SSS5      u  pU R                  SU5        U R                  S	U5        [        SS
-  S-   SS-  S5      u  pU R                  SS-  S-   U5        U R                  SU5        [        SS
-  S-   SS5      u  pU R                  SU5        U R                  SS-  S-   U5        [        SSS5      u  pU R                  SU5        U R                  SU5        g)z%Tests the _GetPartitionInfo function.i,     
      i-     i+                 d   i  3   2   N)r"   assertEqual)selfnum_componentscomponent_sizes      5platform/gsutil/gslib/tests/test_copy_helper_funcs.pytestGetPartitionInfo TestCpFuncs.testGetPartitionInfoL   s    (9c2'F$^R(R( (9c2'F$^R(R( (9c2'F$^R(R( (9a'D$^Q'S.) (9"c'QC9;(=$^b#g]N3R( (9"c'QB'O$^R(b#g]N3 (9c2'F$^Q'R(    c                    [        5       nU R                  S5      nU R                  SSS9n[        R                  " 5       nSnSnU R                  SSS9n[        [        U5      5      n[        U R                  < S	U< S
U< 35      n	[        US5       n
[        U
5      nSSS5        UR                  [        R                  " UUWS9SS9  [        USSUU	SXVX4SS5      nU R                  SSS9n[        [        U5      5      n[        U R                  < S	U< S
U< 35      n[        USSUUSXVUUSS5      nU R                  SSS9n[        [        U5      5      n[        U R                  < S	U< S
U< 35      n[        U R                  SS9S5       n
[        U
5      nSSS5        UR                  [        R                  " UUWS9SS9  [        USSUUSXVX4SS5      nU R                  SSS9n[        [        U5      5      n[        USSUSSXVX4SS5      nU R                  SSS9n[        US5       n
[        U
5      nSSS5        UR                  [        R                  " USWS9SS9  X|UUUUUU0n[        US5      [        US5      [        US5      [        US5      /n[        U R                  < S	U< 35      n[        UUUU5      u  nnn U V!s/ s H  n!U!S   PM
     nn!UUU4 H  n"U R!                  U"U;   5        M     U R#                  S[%        U5      5        U R#                  UR&                  R(                  US   R(                  5        U R#                  S[%        U 5      5        [        U R                  < S	U< S
U< 35      n#U R#                  U#R(                  U S   R(                  5        g! , (       d  f       GN"= f! , (       d  f       GN2= f! , (       d  f       GN= fs  sn!f )z0Tests upload with a variety of component states.bucketfoo   asdf	file_namecontentsContentTypeStorageClassfoo1   1:///rbNrF   namemd5HashrK   r   r8    Ffoo2   2foo4   4   _foo5   5foo6   6)r   MakeTempNameCreateTempFiler   
CreateLockr   strdefault_provideropen#_CalculateB64EncodedMd5FromContentsMockCreateObjectWithMetadataapitools_messagesObjectr(   r   r&   
assertTruer=   lendst_url
url_string)$r>   mock_apibucket_nametracker_filetracker_file_lockcontent_typestorage_classfpath_uploaded_correctlyfpath_uploaded_correctly_urlobject_uploaded_correctly_urlf_infpath_uploaded_correctly_md5args_uploaded_correctlyfpath_not_uploadedfpath_not_uploaded_urlobject_not_uploaded_urlargs_not_uploadedfpath_wrong_contentsfpath_wrong_contents_urlobject_wrong_contents_urlfpath_wrong_contents_md5args_wrong_contentsfpath_remote_deletedfpath_remote_deleted_urlargs_remote_deletedfpath_no_longer_usedfile_md5dst_argsexisting_components
bucket_urlcomponents_to_uploaduploaded_componentsexisting_objects_to_deleteiargno_longer_used_urls$                                       rA   (testFilterExistingComponentsNonVersioned4TestCpFuncs.testFilterExistingComponentsNonVersionedr   s   ~H##H-K&&&IL2==? !L"M  $22V<@  3  B#7$%$' $8			-E	G%H! 
&	-%H%N" 
.))*;*B*B%,+. 48	 * 9 D !Q(D%r<u6 ,,v,M1#6H2IJ2--{<NOQ=Aq"8\,4(  ..$.O3C8L4MN 4			-A	C!D 
d!!4!0$	74!DT!J 
8))*;*B*B!(+* 48	 * 9 @a$<!2|u6  ..$.O3C8L4MN?a$<b"\d  ..$.O	"D	)T4T:h 
*))*;*B*B+;37 * 9
 	!-11	H 	2B7.3.3.3	 &'+'<'<k'K LJ %=	&
H%>!.)<=)<A1Q4)<=!#68KL
ooc112 MQ/01,44??(+668Q678-			-A	CD 	'22/2==?u 
.	-> 
8	7. 
*	)4 >s*   N<O7O O2<
O
O 
O/c                    [        5       nU R                  S5      nUR                  U5        SnSnU R                  SSS9n[        R
                  " 5       nU R                  SSS9n[        [        U5      5      n[        US	5       n	[        U	5      n
S
S
S
5        UR                  [        R                  " UUW
S9SS9n[        U R                  < SU< SU< SUR                  < 35      n[        USSUXR                  X4XVS
S5      nUn[        [        U5      5      nUR                  [        R                  " UUU
S9SS9n[        U R                  < SU< SU< SUR                  < 35      n[        USSUUUR                  X4XVS
S5      nU R                  SSS9n[        [        U5      5      n[        U R                  SS9S	5       n	[        U	5      nS
S
S
5        UR                  [        R                  " UUWS9SS9n[        U R                  < SU< SU< SUR                  < 35      n[        USSUUSX4UUS
S5      nX}UU0n[!        UUR                  5      [!        UUR                  5      [!        UUR                  5      /n[        U R                  < SU< 35      n[#        UUUU5      u  nnnU Vs/ s H  nUS   PM
     nnU R%                  U/U5        U R%                  UR&                  R(                  US   R(                  5        UR&                  R*                  UR&                  R                  4UR&                  R*                  UR&                  R                  4/n U H-  n!U R-                  U!R*                  U!R                  4U ;   5        M/     U R%                  [/        U 5      [/        U5      5        g
! , (       d  f       GN;= f! , (       d  f       GN= fs  snf )z2Tests upload with versionined parallel components.rF   rL   rM   rG   rH   rI   rN   rO   rR   NrS   rV   rP   rQ   #r   r8   FrZ   r[   r\   rW   )r   ra   MockCreateVersionedBucketrb   r   rc   r   rd   rf   rg   rh   ri   rj   re   
generationr(   r   r&   r=   rm   rn   object_namerk   rl   )"r>   ro   rp   rs   rt   rq   rr   ru   rv   rx   ry   object_uploaded_correctlyrw   rz   fpath_duplicatefpath_duplicate_urlduplicate_uploaded_correctly duplicate_uploaded_correctly_urlargs_duplicater   r   r   object_wrong_contentswrong_contents_urlr   r   r   r   r   r   r   r   expected_to_deleteuris"                                     rA   %testFilterExistingComponentsVersioned1TestCpFuncs.testFilterExistingComponentsVersioned   sE    ~H##H-K&&{3 !L"M&&&IL2==?  $22V<@  3  B#7$%$' 	&	-%H%N" 
. ( E E  &>)E	G 	 !F !
 %9			-E	"	-	-	/%0! D !Q(D%'K'K\d	 /O.s?/CD#+#H#H  &5)E	G 	 $I $ 
 (<			-E	%	0	0	2(3$ ;A2($//u	6N  ..$.O3C8L4MN	d!!4!0$	74!DT!J 
8$AA  &:)A	C 	 B 
 .			-A		)	)	+, @a$<B\4( 	!1H 	27BB	D/:EE	G.0B0M0MN &'+'<'<k'K LJ %=	&
H%>!.)<=)<A1Q4)<=)*,@A,44??(+668.66BB.66AAC)11==)11<<>? *
oos7;MMN *S+,c2L.MNg 
.	-L 
8	7F >s   OO?O*
O
O'c                     [         R                  " [        S9n[        S5      n[	        S/5         [
        R                  " X!SS9  S S S 5        UR                  R                  SS/SS	9  g ! , (       d  f       N+= f
Nspecgs://bucketGSUtil0trigger_reauth_challenge_for_parallel_operationsTruer5   worker_countrF   locationgs)fieldsprovider)	r,   Mockr
   r   r   r   .TriggerReauthForDestinationProviderIfNecessary	GetBucketassert_called_once_withr>   ro   destination_urls      rA   testReauthChallengeIsPerformed*TestCpFuncs.testReauthChallengeIsPerformedJ  st    yyh'H*=9O	N 
 @@
!5
 ..x7Al8< / >
 
s   A++
A9c                     [         R                  " [        S9n[        S5      n[        R
                  " UUSS9  UR                  R                  5         g )Nr   r   r5   r   )r,   r   r
   r   r   r   r   assert_not_calledr   s      rA   *testReauthChallengeIsNotPerformedByDefault6TestCpFuncs.testReauthChallengeIsNotPerformedByDefaultX  sE    yyh'H*=9O>>?GLMO ((*rD   c                     [         R                  " [        S9n[        S5      n[	        S/5         [
        R                  " X!SS9  S S S 5        UR                  R                  5         g ! , (       d  f       N)= f)Nr   zdir/filer   r5   r   	r,   r   r
   r   r   r   r   r   r   r   s      rA   2testReauthChallengeNotPerformedWithFileDestination>TestCpFuncs.testReauthChallengeNotPerformedWithFileDestinationb  sc    yyh'H*:6O	N 
 @@
!5
 ((*
 
   A))
A7c                     [         R                  " [        S9n[        S5      n[	        S/5         [
        R                  " X!SS9  S S S 5        UR                  R                  5         g ! , (       d  f       N)= f)Nr   zgs://bucket*r   r5   r   r   r   s      rA   >testReauthChallengeNotPerformedWhenDestinationContainsWildcardJTestCpFuncs.testReauthChallengeNotPerformedWhenDestinationContainsWildcardn  sc    yyh'H*>:O	N 
 @@
!5
 ((*
 
r   c                     [         R                  " [        S9n[        S5      n[	        S/5         [
        R                  " X!SS9  S S S 5        UR                  R                  5         g ! , (       d  f       N)= f)Nr   r   r   r8   r   r   r   s      rA   6testReauthChallengeNotPerformedWithSequentialExecutionBTestCpFuncs.testReauthChallengeNotPerformedWithSequentialExecutionz  sc    yyh'H*=9O	N 
 @@
!5

 ((*
 
r   c                    [         R                  " [        S9n[        R                  R                  S5      UR                  l        [        S5      n[        S/5         U R                  [        R                  R
                  S5         [        R                  " X!SS9  S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)Nr   zReauth errorr   r   r5   r   )r,   r   r
   pyu2ferrorsPluginErrorr   side_effectr   r   assertRaisesRegexr   r   r   s      rA   $testReauthChallengeRaisesReauthError0TestCpFuncs.testReauthChallengeRaisesReauthError  s    yyh'H%*\\%=%=n%MH"*=9O	N 
 !!%,,":":NKBBA	7 L
 
 LK
 
s$   +C	B0C0
B>	:C
Cc                 "   [         R                  " [        S9n[        UR                  l        [        S5      n[        S/5         [        R                  " X!SS9  S S S 5        UR                  R                  SS/SS	9  g ! , (       d  f       N+= fr   )r,   r   r
   	Exceptionr   r   r   r   r   r   r   r   s      rA   &testReauthChallengeSilencesOtherErrors2TestCpFuncs.testReauthChallengeSilencesOtherErrors  s    yyh'H%.H"*=9O	N 
 @@
!5
 ..x7Al8< / >
 
s   B  
Bc                 v   [        [        [        S5      [        5       5      nSUR                  l        [        R                  " SS0SS5      nUR                  U5      nU R                  [        U[        5      5        SUR                  l        [        R                  " SS0SS5      nUR                  U5      nU R                  [        U[        5      5        SUR                  l        [        R                  " SS0SS5      nUR                  U5      nU R                  [        U[        5      5        [        R                  " SS0SS5      nUR                  U5      nU R                  [        U[        5      5        [        R                  " SS	0SS5      nUR                  U5      nU R                  [        U[        5      5        [        R                  " SS
0SS5      nUR                  U5      nU R                  [        U[        5      5        [        R                  " S5      nUR                  U5      nU R                  [        U[        5      5        [        R                  " S5      nUR                  U5      nU R                  [        U[        5      5        U R!                  SUR"                  5        g)zFTests that _TranslateApitoolsResumableUploadException works correctly.	copy_testTstatusi  NFi  i  i  i  zAborting transferzadditional bytes left in streamz&this can happen if a file changes size)r   r   r   r   http"disable_ssl_certificate_validationapitools_exceptions	HttpError*_TranslateApitoolsResumableUploadExceptionrk   
isinstancer   r   r   r   TransferErrorassertInreasonr>   
gsutil_apiexctranslated_excs       rA   -testTranslateApitoolsResumableUploadException9TestCpFuncs.testTranslateApitoolsResumableUploadException  s9   23K@024J :>JOO6

'
'3t
DCJJ3ONOOJ~/?@A9>JOO6

'
'3t
DCJJ3ONOOJ~/GHI9>JOO6

'
'3t
DCJJ3ONOOJ~/GHI

'
'3t
DCJJ3ONOO>#DEG 
'
'3t
DCJJ3ONOO>#DEG 
'
'3t
DCJJ3ONOOJ~/LMN

+
+,?
@CJJ3ONOOJ~/LMN

+
+,M
NCJJ3ONOOJ~/LMNMM: '')rD   c                    [        [        [        S5      [        5       5      n[        R
                  " S5      nUR                  U5      nU R                  [        U[        5      5        U R                  SUR                  5        g)z-Test that StreamExhausted error gets handled.r   zNot enough bytesz]if this issue persists, try deleting the tracker files present under ~/.gsutil/tracker-files/N)r   r   r   r   r   StreamExhaustedr   rk   r   r   r   r   r   s       rA   <testTranslateApitoolsResumableUploadExceptionStreamExhaustedHTestCpFuncs.testTranslateApitoolsResumableUploadExceptionStreamExhausted  so    23K@024J 
-
-.@
ACJJ3ONOOJ~/LMNMM	23A3H3HJrD   c                 Z   [         R                  (       a  [        R                  " S5      $ SnU R	                  5       nU R                  UUSS9nSn[        R                  " X2[        R                  R                  -   U-   5        [        R                  " SS9n[        R                  " U[        R                  R                  -   U-   S9nS	UR                  l        S
UR                  l        S
UR                  l        [!        S/5         [#        Xe5        SSS5        U R%                  SUR&                  5        [        R                  " SS9n[!        S/5         [#        Xe5        SSS5        U R%                  SUR&                  5        g! , (       d  f       N{= f! , (       d  f       N<= f)z=Tests that content type is correctly determined for symlinks.z/use_magicfile features not available on Windowss:   <html><body>And you thought I was just text!</body></html>zhtml_in_disguise.txt)tmpdirrK   rJ   link_to_realfileNcontentTyper   TF)r   use_magicfiler   ztext/html; charset=us-ascii)r   r   Falsez
text/plain)r   
IS_WINDOWSr   skipCreateTempDirrb   ossymlinkpathsepr,   	MagicMock	IsFileUrlreturn_valueIsStreamIsFifor   r$   r=   r   )r>   surprise_htmltemp_dir_pathtxt_file_path	link_namedst_obj_metadata_mocksrc_url_stubs          rA   testSetContentTypeFromFile&TestCpFuncs.testSetContentTypeFromFile  s[   ]]LMMQM&&(M''}1>2H ( JM #IJJ}bggkk9IEF NNt<>>mbggkk.I"+/, -L*.L').L&',L$ 
BC	DlB 
E2*668 !NNt<	CD	ElB 
F\#8#D#DE 
E	D 
F	Es   
FF
F
F*c                 ^   [         R                  R                  5       nU H  u  p#[        R                  " S S9n[        R                  " SU-   S9nSUR
                  l        SUR                  l        SUR                  l        [        XT5        U R                  X4R                  5        M     g )Nr   zfile.r   TF)r   COMMON_EXTENSION_RULESitemsr,   r   r   r   r   r   r$   r=   r   )r>   extension_rules	extensionexpected_content_typer  r  s         rA   4testSetsContentTypesForCommonFileExtensionsCorrectly@TestCpFuncs.testSetsContentTypesForCommonFileExtensionsCorrectly  s    !88>>@O,;(	"nn>^^)0CDl,0l)+0l().l&lB
,.O.OP -<rD   i           	      z	time.time)r   )newc           
         [         R                  " S5      n[        S5      nU R                  U R                  [        R
                  " SSS9-
  4 Hk  n[        R                  " SUS9n[        R                  R                  US5       n[        X$U5        UR                  S	S
UR                  S5        SSS5        Mm     U R                  U R                  [        R
                  " SSS9-
  4 Hk  n[        R                  " SUS9n[        R                  R                  US5       n[        X$U5        UR                  S	SUR                  S5        SSS5        Mm     U R                  U R                  [        R
                  " SSS9-
  4 Hk  n[        R                  " SUS9n[        R                  R                  US5       n[        X&U5        UR                  S	SUR                  S5        SSS5        Mm     [        R                  R                  US5       n[        R                  " SU R                  [        R
                  " SSS9-
  S9n[        X'U5        UR                  5         SSS5        [        R                  R                  US5       n[        R                  " SU R                  [        R
                  " SSS9-
  S9n[        X(U5        UR                  5         SSS5        [        R                  R                  US5       n[        R                  " SU R                  [        R
                  " SSS9-
  S9n	[        X)U5        UR                  5         SSS5        [        R                  R                  US5       n[        R                  " SU R                  S9n
[        X*U5        UR                  5         SSS5        g! , (       d  f       GM  = f! , (       d  f       GM  = f! , (       d  f       GMx  = f! , (       d  f       GN= f! , (       d  f       GN9= f! , (       d  f       N= f! , (       d  f       g= f)zCTests that WarnIfEarlyDeletionChargeApplies warns when appropriate.testzgs://bucket/object      )dayshoursNEARLINE)storageClasstimeCreatedwarnzWarning: moving %s object %s may incur an early deletion charge, because the original object is less than %s days old according to the local system time.nearliner3   NY   COLDLINEcoldlineZ   il  ARCHIVEarchiveim  r8   )r  secondsSTANDARD)loggingLoggerr   _PI_DAYdatetime	timedeltari   rj   r,   patchobjectr)   assert_called_withrn   r   )r>   test_loggersrc_urlobject_time_createdrecent_nearline_objmocked_warnrecent_archive_objold_nearline_objold_coldline_objold_archive_objnot_old_enough_nearline_objs              rA   &testWarnIfMvEarlyDeletionChargeApplies2TestCpFuncs.testWarnIfMvEarlyDeletionChargeApplies  s   
 ..(K"#78G !%dll ( 2 2" E/F  G-44!/BD ::[&1[*7+6	8&&23=		$ 21 G !%dll ( 2 2" E/F  G-44!/BD ::[&1[*7+6	8&&23=		$ 21 G !%dll ( 2 22 F/G  H,33 .AC ::[&1[*7+6	8&&23<		% 21 H 
		;	/;*11!llX%7%7R%KKM )KP##% 
0 
		;	/;*11!llX%7%7R%KKM )KP##% 
0 
		;	/;)00 llX%7%7S!%LLNo );O##% 
0 
		;	/;$5$<$<!t||%=!()46##% 
0	/y 21 21 21 
0	/ 
0	/ 
0	/ 
0	/sW   +O%+O+O2APAPAP)<P:
O	
O/	2
P	
P
P&)
P7:
Qc                     Sn[         nU Hg  n[        USUS5      u  pEU R                  U5        U R                  U5        [        USUS5      u  pEU R                  U5        U R                  U5        Mi     g )N)zfile://testtest.xmlztest.pyFT)r'   r#   rk   assertFalse)r>   pathsextsr   zippedgzip_encodeds         rA   &testSelectUploadCompressionStrategyAll2TestCpFuncs.testSelectUploadCompressionStrategyAllY  sv    2ED=
tU$f
oof
|$=
tT#f
v
ool# rD   c                     [        SSS/S5      u  pU R                  U5        U R                  U5        [        SSS/S5      u  pU R                  U5        U R                  U5        g )Nr?  Fxmlyaml)r#   rk   r@  r>   rC  rD  s      rA   )testSelectUploadCompressionStrategyFilter5TestCpFuncs.testSelectUploadCompressionStrategyFilterf  sh    ;EE7E+FOOF\";EF8U,FV\"rD   c                 j    [        SSS/S5      u  pU R                  U5        U R                  U5        g )Nr?  Tnot_matching)r#   r@  rk   rJ  s      rA   ,testSelectUploadCompressionStrategyComponent8TestCpFuncs.testSelectUploadCompressionStrategyComponentp  s5    ;D>*D2FVOOL!rD   c           	          [         R                  " 5       n[         R                  " 5       Ul        S n[        USUSSSS 5      u  p4U R	                  US5        U R	                  US5        U R                  UR                  R                  5        g )Nc                      gN)ab rV  rD   rA   DelegateUploadHTestCpFuncs.testDelegateUploadFileToObjectNormal.<locals>.DelegateUploadz      rD   urlFrT  rU  )r,   r   closer!   r=   rk   called)r>   mock_streamrW  elapsed_timeuploaded_objects        rA   $testDelegateUploadFileToObjectNormal0TestCpFuncs.testDelegateUploadFileToObjectNormalv  sz    ))+K		K %@{E5%%G!L 	\3'_c*OOK%%,,-rD   z	os.unlinkc           	         [         R                  " 5       n[         R                  " 5       Ul        [         R                  " 5       nSUl        S n[	        XCUSSSS 5      u  pVU R                  US5        U R                  US5        U R                  UR                  5        U R                  UR                  R                  5        g )NSamplec                      grS  rV  rV  rD   rA   rW  HTestCpFuncs.testDelegateUploadFileToObjectZipped.<locals>.DelegateUpload  rY  rD   TFrT  rU  )r,   r   r[  r   r!   r=   rk   r\  )r>   mock_unlinkr]  mock_upload_urlrW  r^  r_  s          rA   $testDelegateUploadFileToObjectZipped0TestCpFuncs.testDelegateUploadFileToObjectZipped  s    ))+K		KiikO"*O %@dE5$%P!L 	\3'_c*OOK&&'OOK%%,,-rD   z/gslib.command.concurrent_compressed_upload_lockc           	      v  ^ ^ [         R                  " 5       n[         R                  " 5       Ul        UU 4S jn[        USUSSSS 5      u  pET R	                  US5        T R	                  US5        T R                  UR                  R                  5        T R                  TR                  R                  5        g )Nc                  P   > TR                  T R                  R                  5        grS  )rk   	__enter__r\  	mock_lockr>   s   rA   rW  MTestCpFuncs.testDelegateUploadFileToObjectGzipEncoded.<locals>.DelegateUpload  s    
ooi))001rD   rZ  FTrT  rU  )r,   r   r[  r!   r=   rk   r\  __exit__r>   rn  r]  rW  r^  r_  s   ``    rA   )testDelegateUploadFileToObjectGzipEncoded5TestCpFuncs.testDelegateUploadFileToObjectGzipEncoded  s    ))+K		K
 %@{E4%F!L 	\3'_c*OOK%%,,-OOI&&--.rD   c           	      v  ^ ^ [         R                  " 5       n[         R                  " 5       Ul        UU 4S jn[        USUSSSS 5      u  pET R	                  US5        T R	                  US5        T R                  UR                  R                  5        T R                  TR                  R                  5        g )Nc                  P   > TR                  T R                  R                  5        grS  )r@  rl  r\  rm  s   rA   rW  VTestCpFuncs.testDelegateUploadFileToObjectGzipEncodedComposite.<locals>.DelegateUpload  s     
y**112rD   rZ  FTrT  rU  )	r,   r   r[  r!   r=   rk   r\  r@  rp  rq  s   ``    rA   2testDelegateUploadFileToObjectGzipEncodedComposite>TestCpFuncs.testDelegateUploadFileToObjectGzipEncodedComposite  s    ))+K		K
 %@{E4t%E!L 	\3'_c*OOK%%,,-Y''../rD   c                 V    [         R                  " SSSSS9nU R                  SU5        g )NTF)file_size_will_changesize)r   GetSourceFieldsNeededForCopyassertNotIn)r>   r   s     rA   1testDoesNotGetSizeSourceFieldIfFileSizeWillChange=TestCpFuncs.testDoesNotGetSizeSourceFieldIfFileSizeWillChange  s+    55dE7FVV$rD   rV  N)$__name__
__module____qualname____firstlineno____doc__rB   r   r   r   r   r   r   r   r   r   r   r   r  r  r-  r,  r,   r/  r   r   ConvertDatetimeToPOSIXr<  rE  rK  rO  r`  rh  rr  rw  r~  __static_attributes__rV  rD   rA   r.   r.   I   s#   ?$)Ln?`fOP>+
+
+	+
7> *)X
JF@Q dAr2q"5'::k..)@@IKLL&LL&\$#". ::k. .& ::?@/ A/& ::?@0 A0&%rD   r.   c                   V   \ rS rSr\R
                  " S5      \R
                  " S5      S 5       5       r\R
                  " S5      \R
                  " S5      S 5       5       r\R
                  " S5      \R
                  " S5      S 5       5       r\R
                  " S5      \R
                  " S5      S 5       5       r	\R
                  " S5      \R
                  " S5      S	 5       5       r
\R
                  " S5      \R
                  " S5      S
 5       5       r\R
                  " S5      S 5       r\R
                  " S5      S 5       r\R
                  " S5      S 5       r\R
                  " S5      S 5       r\R
                  " S5      S 5       r\R
                  " S5      S 5       rS rSrg)TestExpandUrlToSingleBlri  zgslib.cloud_api.CloudApiz.gslib.utils.copy_helper.CreateWildcardIteratorc           	          [        S5      n[        [        U5      /5      Ul        [	        SUSS[        S5      5      u  pEU R                  U5        U R                  XC5        g Ngs://test/helloworldgs://test/hello*/
project_idFr   )r   iterr	   r   r%   r   rk   r=   r>   mock_CreateWildcardIteratormock_gsutil_apistorage_urlexp_urlhave_existing_dst_containers         rA   $testContainsWildcardMatchesNotObject=TestExpandUrlToSingleBlr.testContainsWildcardMatchesNotObject  sc     ''=>K/3	[	)*0,,-A_lE,..*W 	OO/0W*rD   c           	          [        S5      n[        [        U5      /5      Ul        [	        SUSS[        S5      5      u  pEU R                  U5        U R                  XC5        g r  )r   r  r   r   r%   r   r@  r=   r  s         rA   !testContainsWildcardMatchesObject:TestExpandUrlToSingleBlr.testContainsWildcardMatchesObject  se     ''=>K/3	[	)*0,,-A_lE,..*W 	01W*rD   c           
          [        [        [        S5      5      [        [        S5      5      /5      Ul        U R	                  [
        5         [        SUSS[        S5      5        S S S 5        g ! , (       d  f       g = f)Nr  zgs://test/helloworld2r  r  Fr   )r  r   r   r   assertRaisesr   r%   r   r>   r  r  s      rA   #testContainsWildcardMultipleMatches<TestExpandUrlToSingleBlr.testContainsWildcardMultipleMatches  sm     0401GHI01HIJ5 0, 
		+	,. "9+"FH 
-	,	,s   A..
A<c           
          [        / 5      Ul        U R                  [        5         [	        SUSS[        S5      5        S S S 5        g ! , (       d  f       g = f)Nr  r  Fr   )r  r   r  r   r%   r   r  s      rA   testContainsWildcardNoMatches6TestExpandUrlToSingleBlr.testContainsWildcardNoMatches  sG     04Bx,			+	,. "9+"FH 
-	,	,s   A
Az,gslib.utils.copy_helper.StorageUrlFromStringc           	          [         R                  " 5       nSUR                  l        SUR                  l        X1l        [        SUSS[        S5      5      u  pEU R                  U5        U R                  XC5        g )NT
/home/testr  Fr   )	r,   r   	isFileUrlr   IsDirectoryr%   r   rk   r=   r>   mock_StorageUrlFromStringr  mock_storage_urlr  r  s         rA   testLocalFileDirectory/TestExpandUrlToSingleBlr.testLocalFileDirectory  so     yy{.2+04  --=*-Ao|U,..*W 	OO/0W/rD   c           	          [         R                  " 5       nSUR                  l        SUR                  l        X1l        [        SUSS[        S5      5      u  pEU R                  U5        U R                  XC5        g )NTFr  r  r   )	r,   r   r  r   r  r%   r   r@  r=   r  s         rA   testLocalFileNotDirectory2TestExpandUrlToSingleBlr.testLocalFileNotDirectory  sq     yy{.2+05  --=*-Ao|U,..*W 	01W/rD   c           	          [        [        R                  " S[        R                  R                  5      /5      UR
                  l        [        SUSS[        S5      5      u  p#U R                  U5        U R                  U[        S5      5        g )Nfolder/gs://test/folderr  Fr   r  r
   CsObjectOrPrefixCsObjectOrPrefixTypePREFIXListObjectsr   r%   r   rk   r=   r   r>   r  r  r  s       rA   testNoSlashPrefixExactMatch4TestExpandUrlToSingleBlr.testNoSlashPrefixExactMatch  s    /3!!)"*"?"?"F"F	H5 0O, .BO\5,..*W 	OO/0W23EFGrD   c           	          [        [        R                  " S[        R                  R                  5      /5      UR
                  l        [        SUSS[        S5      5      u  p#U R                  U5        U R                  U[        S5      5        g )Nz
folderone/r  r  Fr   )r  r
   r  r  r  r  r   r%   r   r@  r=   r   r  s       rA   testNoSlashPrefixSubstringMatch8TestExpandUrlToSingleBlr.testNoSlashPrefixSubstringMatch$  s    /3!!,"*"?"?"F"F	H5 0O, .BO\5,..*W 	01W23EFGrD   c           	      D   [        [        R                  " [        R                  " SS9[        R
                  R                  5      /5      UR                  l        [        SUSS[        S5      5      u  p#U R                  U5        U R                  U[        S5      5        g )Nzfolder_$folder$)rT   r  r  Fr   )r  r
   r  ri   rj   r  OBJECTr  r   r%   r   rk   r=   r   r  s       rA   testNoSlashFolderPlaceholder5TestExpandUrlToSingleBlr.testNoSlashFolderPlaceholder1  s    /3!!$$*;<))00	25 0O,
 .BO\5,..*W 	OO/0W23EFGrD   c           	          [        / 5      UR                  l        [        SUSS[	        S5      5      u  p#U R                  U5        U R                  U[        S5      5        g )Nr  r  Fr   )r  r  r   r%   r   r@  r=   r   r  s       rA   testNoSlashNoMatch+TestExpandUrlToSingleBlr.testNoSlashNoMatch?  sZ    /3BxO,-AO\5,..*W 	01W23EFGrD   c           	          [        [        R                  " S[        R                  R                  5      /5      UR
                  l        [        SUSS[        S5      5      u  p#U R                  U5        U R                  U[        S5      5        g )Nr  gs://test/folder/r  Fr   r  r  s       rA   testWithSlashPrefixExactMatch6TestExpandUrlToSingleBlr.testWithSlashPrefixExactMatchI  s    /3!!)"*"?"?"F"F	H5 0O, .B_lE,..*W 	OO/0W23FGHrD   c           	          [        / 5      UR                  l        [        SUSS[	        S5      5      u  p#U R                  U5        U R                  U[        S5      5        g )Nr  r  Fr   )r  r  r   r%   r   rk   r=   r   r  s       rA   testWithSlashNoMatch-TestExpandUrlToSingleBlr.testWithSlashNoMatchV  sX    /3BxO,-A_lE,..*W 	OO/0W23FGHrD   c                     [         R                  " SS/5      n[        S/5         [        S S S U" SS9U" SS9S9  S S S 5        g ! , (       d  f       g = f)N
FakeObjectrU   )r   check_hashesneverrT  )rU   rU  )loggerr3  rm   src_obj_metadatadst_obj_metadata)collections
namedtupler   r    )r>   r  s     rA   &testCheckCloudHashesIsSkippedCorrectly?TestExpandUrlToSingleBlr.testCheckCloudHashesIsSkippedCorrectly`  sP    ''ykBJ	BC	Dt $ $)3C)@)3C)@	B 
E	D	Ds   A
ArV  N)r  r  r  r  r,   r/  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rV  rD   rA   r  r    s   ::()::>?
+ @ *
+ ::()::>?
+ @ *
+ ::()::>?H @ *H ::()::>?H @ *H ::()::<=
0 > *
0 ::()::<=0 > *0 ::()
H *
H ::()
H *
H ::()H *H ::()H *H ::()
I *
I ::()I *I	BrD   r  )Hr  
__future__r   r   r   r   r  r-  r*  r   r   apitools.base.pyr   r   gslib.bucket_listing_refr   r	   gslib.cloud_apir
   r   r   r   r   gslib.commandr   gslib.discard_messages_queuer   gslib.exceptionr   gslib.gcs_json_apir   gslib.parallel_tracker_filer   gslib.storage_urlr   gslib.tests.mock_cloud_apir   "gslib.tests.testcase.unit_testcaser   gslib.tests.utilr   r   r   "gslib.third_party.storage_apitoolsr   ri   gslib.utilsr   r   r   r   r   gslib.utils.copy_helperr    r!   r"   r#   r$   r%   r&   r'   r(   r)   sixr*   r+   	six.movesr,   "CalculateB64EncodedMd5FromContentsrg   r.   r  rV  rD   rA   <module>r     s    ? & %  '    	  > 8 8 $ 9 4 = , 1 = , ) 9 2 3 A 3 1 % W # 2 " # & 5 ? 5 D ; 8 < 2 I F % VV_	5 6  55 ${	%$ {	%|bB1 bBrD   