
    O4                     x   S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKJr  SSK	J
r
  SS	K	Jr  SS
K	Jr  SSKJs  Jr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJ r    " S S\RB                  5      r" " S S\RF                  5      r$ " S S\RJ                  5      r&g)!Integration tests for mv command.    )absolute_import)print_function)division)unicode_literalsN)ApiSelector) TestCpMvPOSIXBucketToLocalErrors)"TestCpMvPOSIXBucketToLocalNoErrors)"TestCpMvPOSIXLocalToBucketNoErrors)	SkipForS3)ObjectToURI)SequentialAndParallelTransfer)SetBotoConfigForTest)SetEnvironmentForTest)unittest)UsingCrcmodExtension)Retry)
IS_WINDOWS)	shim_utilc                   *    \ rS rSrSrS rS rS rSrg)TestMvUnitTests)   zUnit tests for mv command.c           	         U R                  5       nU R                  USSS9  U R                  USSS9  U R                  5       nU R                  S[        US5      [        U5      /5        [	        S U R                  [        US5      5      R                  S	S
9 5       5      n[	        [        USS5      [        USS5      /5      nU R                  X45        g)6Tests moving multiple top-level items between buckets.dir1/file.txt   data
bucket_uriobject_namecontentsdir2/file.txtmv*c              3   8   #    U  H  n[        U5      v   M     g 7fNstr.0us     &platform/gsutil/gslib/tests/test_mv.py	<genexpr>]TestMvUnitTests.test_move_bucket_objects_with_duplicate_names_inter_bucket.<locals>.<genexpr>9   !      ,+A 	A+   **Texpand_top_level_bucketsdir1file.txtdir2NCreateBucketCreateObject
RunCommandsuriset_test_wildcard_iteratorIterAllassertEqualselfbucket1_uribucket2_uriactualexpecteds        r+   :test_move_bucket_objects_with_duplicate_names_inter_bucketJTestMvUnitTests.test_move_bucket_objects_with_duplicate_names_inter_bucket,   s    ##%K"1&  ( 	"1&  ( ##%KOOD4S143DEF ,--d;.EFNN%) O +, ,F [&*-[&*- H 	V&    c           	         U R                  5       nU R                  USSS9  U R                  USSS9  U R                  5       nU R                  S[        US5      [        US5      /5        [	        S U R                  [        US	5      5      R                  S
S9 5       5      n[	        [        USSS5      [        USSS5      /5      nU R                  X45        g)r   r   r   r   r!   r"   r#   dirc              3   8   #    U  H  n[        U5      v   M     g 7fr%   r&   r(   s     r+   r,   aTestMvUnitTests.test_move_bucket_objects_with_duplicate_names_to_bucket_subdir.<locals>.<genexpr>P   r.   r/   r0   Tr1   r3   r4   r5   Nr6   r?   s        r+   >test_move_bucket_objects_with_duplicate_names_to_bucket_subdirNTestMvUnitTests.test_move_bucket_objects_with_duplicate_names_to_bucket_subdirC   s    ##%K"1&  ( 	"1&  ( ##%KOOD4S14U3KLM ,--d;.EFNN%) O +, ,F [%4[%4 H 	V&rG   c           	      ^   U R                  5       nU R                  SS/S9nU R                  SUS-   [        U5      /5        [	        S U R                  [        US5      5      R                  SS	9 5       5      n[	        [        US
S5      [        USS5      /5      nU R                  X45        g)2Tests moving multiple top-level items to a bucket.)r3   r4   )r5   r4   
test_filesr"   z/*c              3   8   #    U  H  n[        U5      v   M     g 7fr%   r&   r(   s     r+   r,   UTestMvUnitTests.test_move_dirs_with_duplicate_file_names_to_bucket.<locals>.<genexpr>d   r.   r/   r0   Tr1   r3   r4   r5   N)r7   CreateTempDirr9   r:   r;   r<   r=   r>   )r@   r   dir_pathrC   rD   s        r+   2test_move_dirs_with_duplicate_file_names_to_bucketBTestMvUnitTests.test_move_dirs_with_duplicate_file_names_to_bucketZ   s    ""$J!!.! H
 	OOD8d?D,<=> ,--d:t.DEMM%) N +, ,F Z,Z, H 	V&rG    N)	__name__
__module____qualname____firstlineno____doc__rE   rL   rV   __static_attributes__rX   rG   r+   r   r   )   s    "'.'.'rG   r   c                       \ rS rSrSrS rSrg)TestMvUnitTestsWithShimo   z$Unit tests for mv command with shim.c                    U R                  5       nU R                  SS9n[        SS/5         [        SSS.5         U R	                  SS	S
SSSU[        U5      /SS9nSR                  UR                  S   5      nU R                  SR                  [        R                  " S5      U[        U5      5      U5        S S S 5        S S S 5        g ! , (       d  f       N= f! , (       d  f       g = f)Ns   abcdr    )GSUtiluse_gcloud_storageTrue)rd   hidden_shim_modedry_runrf   fake_dir)(CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTILCLOUDSDK_ROOT_DIRr"   z-azpublic-readz-Uz-rz-RT)return_log_handler
infozZGcloud Storage Command: {} storage mv --predefined-acl publicRead --skip-unsupported {} {})r7   CreateTempFiler   r   r9   r:   joinmessagesassertInformatr   _get_gcloud_binary_path)r@   r   fpathmock_log_handler
info_liness        r+   test_shim_translates_flags2TestMvUnitTestsWithShim.test_shim_translates_flagsr   s    ""$J1E	GHJ 
K 6<)" 	  ??4dD%
#%# + % YY/88@A
DDJF11*=uZ E"#-		/	
K 
K	 	
K 
Ks#   CB C<C
C	C
C,rX   N)rY   rZ   r[   r\   r]   rx   r^   rX   rG   r+   r`   r`   o   s
    ,/rG   r`   c                   .   \ rS rSrSrS rS rS r\S 5       r	S r
\R                  " \S5      \R                  " \" 5       S	5      S
 5       5       r\R                  " \S5      S 5       r\R                  " \S5      S 5       r\" S5      S 5       rS rSrg)TestMvE2ETests   r   c           	         U R                  SS9nU R                  US5        U R                  5       nU R                  US5        UR                  5        Vs/ s H  nU R                  X5      R                  PM      nnSS/U-   [        U5      /-   nU R                  USS9nU R                  UR                  S5      SS	U-  5        U R                  UR                  S5      S
SU-  5        U R                  UR                  S5      S-  SSU-  5        U R                  UR                  S5      SSU-  5        U R                  US5        U R                  US5        UR                  5        Vs/ s H  nU R                  X#5      R                  PM      nnUS   nU R                  SU/5        U R                  US5        U R                  US5        UR                  5        Vs/ s H  n[        U R                  X#5      5      PM     nnSS/U-   [        U5      /-   nU R                  USS9nU R                  UR                  S5      SSU-  5        U R                  UR                  S5      SSU-  5        U R                  UR                  S5      S5        U R                  US5        U R                  US5        gs  snf s  snf s  snf )zDTests moving two buckets, one with 2 objects and one with 0 objects.   test_objectsr   z-mr"   Treturn_stderrCopyingz,stderr did not contain 2 "Copying" lines:
%s   z/stderr did not contain <= 4 "Copying" lines:
%sz9stderr did not contain even number of "Copying" lines:
%sRemovingz-stderr did not contain 2 "Removing" lines:
%srm   z/stderr did not contain >= 1 "Copying" lines:
%sz/stderr did not contain <= 2 "Copying" lines:
%sN)r7   AssertNObjectsInBucketlist_bucketStorageUriCloneReplaceKeyversionless_urir:   	RunGsUtilassertGreaterEqualcountassertLessEqualr>   )r@   rA   rB   keyobjscmdstderrobj1s           r+   test_movingTestMvE2ETests.test_moving   s   ###3KQ/##%KQ/
 **,,C 	&&{8HH, 	  $<$${"3!44C^^Ct^4FY7&@B 	Y:VCE 	Y!#QDvMO 	V\\*-qENP 	Q/Q/
 **,,C 	&&{8HH, 	  7DNND$< Q/Q/
 **,,C 	T++K=>, 	  $<$${"3!44C^^Ct^4FY:VCE 	Y:VCE 	V\\*-q1Q/Q/e.s   %J=.%K#$Kc                    U R                  SS9nU R                  US5        U R                  5       nU R                  S[	        US5      U/5        / n[
        R                  " U5       H=  u  pEnU H1  nUR                  [
        R                  R                  XG5      5        M3     M?     U R                  [        U5      S5        U R                  US5        g)+Tests moving a local directory to a bucket.r~   r   r"   r#   r   N)r7   r   rT   r   r:   oswalkappendpathrp   r>   len)r@   r   tmpdirdir_listdirname_	filenamesfilenames           r+   test_move_bucket_to_dir&TestMvE2ETests.test_move_bucket_to_dir   s    """2J
A.!FNND$z3/89H!#I(W78   "1 	S]A&
A.rG   c                     U R                  5       nU R                  SS9nU R                  SU[        U5      /5        U R	                  US5        g)r   r~   rP   r"   N)r7   rT   r   r:   r   )r@   r   dir_to_moves      r+   test_move_dir_to_bucket&TestMvE2ETests.test_move_dir_to_bucket   sK    ""$J$$$2KNND+tJ'789
A.rG   c                 *  ^ ^^^ T R                  5       nT R                  USS9mT R                  USS9mT R                  5       mT R                  SS[	        T5      /SR                  TT45      S9  [        [        SS	S
9UUUU 4S j5       nU" 5         g)zTests mv with the -I option.   data1)r   r       data2r"   z-Irm   )stdin   r   )triestimeout_secsc                    > TR                  S[        T5      /SS9n TR                  [        R                  R                  T5      U 5        TR                  [        R                  R                  T5      U 5        TR                  U S5        g )NlsTreturn_stdoutr~   )r   r:   rr   r   r   basenameassertNumLines)stdoutr   fpath1fpath2r@   s    r+   _Check1/TestMvE2ETests.test_stdin_args.<locals>._Check1   si    ~~tT*%56d~Kf
mmBGG$$V,f5
mmBGG$$V,f5
&!$rG   N)rT   ro   r7   r   r:   rp   r   AssertionError)r@   r   r   r   r   r   s   `  @@@r+   test_stdin_argsTestMvE2ETests.test_stdin_args   s     !F   BF   BF""$JNND$Z 01FF#34  6 >3% 4% IrG   c                    U R                  SS9nU R                  5       nU R                  USS9nU R                  SSU[	        U5      /SS9nU R
                  (       a  U R                  S	[	        U5      -  U5        OU R                  S
[	        U5      -  U5        U R                  S[	        U5      -  U5        U R                  S[	        U5      /SS9nU R                  US5        g)zTests mv with the -n option.r   rc   r   r   r    r"   z-nTr   z3Skipping existing destination item (no-clobber): %szSkipping existing item: %szRemoving %scatr   data2N)	ro   r7   r8   r   r:   _use_gcloud_storagerr   assertNotInr>   )r@   r   r   
object_urir   r    s         r+   test_mv_no_clobber!TestMvE2ETests.test_mv_no_clobber   s      ( 3F""$J""j8"LJ^^	tVT*-.d  DF 
mm
?
z
"$ mm04
3CCVL]T&\16:~~ud:&67t~LHXw'rG   z*POSIX attributes not available on Windows.zTest requires fast crcmod.c                 Z    U R                  5       nU R                  5       n[        XUSS9  g)zTests use of the -P flag with mv from a bucket to a local dir.

Specifically tests combinations of POSIX attributes in metadata that will
pass validation.
Fis_cpN)r7   rT   r
   )r@   r   r   s      r+   .test_mv_preserve_posix_bucket_to_dir_no_errors=TestMvE2ETests.test_mv_preserve_posix_bucket_to_dir_no_errors  s,     ""$J!F&tuMrG   c                 |    U R                  5       nU R                  5       nU R                  USSS9n[        XX2SS9  g)zTests use of the -P flag with mv from a bucket to a local dir.

Specifically, combinations of POSIX attributes in metadata that will fail
validation.
obj   objr   Fr   N)r7   rT   r8   r	   )r@   r   r   r   s       r+   +test_mv_preserve_posix_bucket_to_dir_errors:TestMvE2ETests.test_mv_preserve_posix_bucket_to_dir_errors  sK     ""$J!F


z(-%+  -C %Ts%PrG   c                 8    U R                  5       n[        XSS9  g)z>Tests use of the -P flag with mv from a local dir to a bucket.Fr   N)r7   r   )r@   r   s     r+   -test_mv_preseve_posix_dir_to_bucket_no_errors<TestMvE2ETests.test_mv_preseve_posix_dir_to_bucket_no_errors  s     ""$J&tuErG   z-Test is only relevant for gs storage classes.c                 2   U R                   [        R                  :X  a  [        R                  " S5      $ U R                  SS9nU R                  USS9nU R                  S[        U5      [        US5      /SS	9nU R                  S
[        U5      -  U5        g)zETests that mv on a recent nearline object warns about early deletion.z)boto does not return object storage classNEARLINE)storage_classr   r   r"   fooTr   zWarning: moving nearline object %s may incur an early deletion charge, because the original object is less than 30 days old according to the local system time.N)
test_apir   XMLr   skipr7   r8   r   r:   rr   )r@   r   r   r   s       r+   test_mv_early_deletion_warning-TestMvE2ETests.test_mv_early_deletion_warning"  s     }}']]FGG"""<J""j6"JJ^^	tJ	j%	 	"15  7F 	MM	.04Z0@	ABHJrG   c                    U R                  5       nU R                  USSS9  U R                  USSS9  U R                  US5        U R                  5       nU R	                  S[        US5      U/5        / n[        R                  " U5       H=  u  pEnU H1  nUR                  [        R                  R                  XG5      5        M3     M?     U R                  [        U5      S5        U R                  SR                  U[        R                  [        R                  5      U5        U R                  S	R                  U[        R                  [        R                  5      U5        U R                  US
5        g)rO   r   r   r   r!   r~   r"   r#   z{}{}dir1{}file.txtz{}{}dir2{}file.txtr   N)r7   r8   r   rT   r   r:   r   r   r   r   rp   r>   r   rr   rs   sep)r@   r   r   	file_listr   r   r   r   s           r+   4test_move_bucket_objects_with_duplicate_names_to_dirCTestMvE2ETests.test_move_bucket_objects_with_duplicate_names_to_dir2  s;   ""$J"1&  ( 	"1&  ( 	
A.!FNND$z3/89I!#I(g89   "1 	S^Q'MM&--fbffbffEMM&--fbffbffE
A.rG   rX   N)rY   rZ   r[   r\   r]   r   r   r   r   r   r   r   skipIfr   
skipUnlessr   r   r   r   r   r   r   r^   rX   rG   r+   r{   r{      s    ):0x// ! !&(& ??:KL+-/KLN M MN ??:KLQ MQ ??:KLF MF
 <=J >J/rG   r{   )'r]   
__future__r   r   r   r   r   gslib.cs_api_mapr   gslib.tests.test_cpr	   r
   r   gslib.tests.testcaseteststestcase)gslib.tests.testcase.integration_testcaser   gslib.tests.utilr   r:   r   r   r   r   gslib.utils.boto_utilr   gslib.utils.retry_utilr   gslib.utils.system_utilr   gslib.utilsr   GsUtilUnitTestCaser   ShimUnitTestBaser`   GsUtilIntegrationTestCaser{   rX   rG   r+   <module>r      s    ( & %  ' 	 ( @ B B ' ' ? 0 : 1 2 % 6 ( . !C'h11 C'L/h77 /0B/X77 B/rG   