
    	t                        S SK Jr  S SK Jr  S SK Jr  S SK Jr  S SKJr  S SKrS SKrS SK	r	S SK
r
S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKJr  S SKJr  S SKrS SKrS SKrS S	KJr  S S
KJr  S SK r S SK!J"r"  S SK!J#r#  S SK$J%r%  S SK&J'r(  S SK)J*r*  S SK+J,r,J-r-  S SK.J/r/  S SK0J1r1  S SK2J3r3  S SK4J5r5  \3(       dm  S SK6r6S SK7r7S r8S r9S r:S r;\*Rx                  " 5         \=" \*R|                  S5      r?\
R                  " 5       rA\%" S 5      rB\%" S 5      rC\%" S 5      rD\%" S 5      rESoS jrFSpS jrGSpS  jrHS!rI\1" \I5      rJS"rK\1" \K5      rLS#rM\1" \M5      rNS$rO\1" \O5      rPS%rQS&rRS'rSS(rTS)rUS*rVS+rWS,rXS-rYS.rZS/r[S0r\S1r]S2r^S3r_S4r`S5raS6rbS6rcSrdSreS7rfS8rgS9rhS:riS;rjS<rk " S= S>\l5      rm\mR                  4S? jroS@ rpSA rq\R                  R                  SBSCS5      =(       a    \R                  R                  SBSDS5      rt\R                  R                  SBSESF5      R                  SG5      rv\-" 5       rw\R                  R                  SBSHS5      SLrx\R                  R                  SBSIS5      SLry\R                  R                  SJSKSL5      R                  5       SM:g  r{SN r|SO r}\}" 5       r~SP rSQ rSR r " SS ST\ GR                  5      r\" 5       r " SU SV\ GR
                  5      rSWrSX rSY rSZ rS[ rS\ r\S] 5       r\SqS^ j5       r\S_ 5       r\S` 5       rSa rSb rSc r\Sd 5       rSe rSf r " Sg Sh\l5      r " Si Sj\l5      r " Sk Sl\l5      r " Sm Sn\5      rg)r    )absolute_import)print_function)division)unicode_literals)contextmanagerN)urllib)	cStringIO)KmsApi)PopulateProjectId)ResumableDownloadException)ResumableUploadException)LazyWrapper)
posix_util)UsingCrcmodExtensionHasUserSpecifiedGsHost)UTF8)#Base64Sha256FromBase64EncryptionKey)
IS_WINDOWS)MakeHumanReadablec                  >   [        [        R                  " 5        V s/ s H  o R                  PM     sn 5      nSnUS:  a0  X!;   a  US-  nM   [        R                  " U5        US-  nUS:  a  M0  [        S5      es  sn f ! [
         a    Us $ [         a     M*  f = f)Ni  l    rT    zUnable to generate GID for )sortedgrpgetgrallgr_gidgetgrgidKeyErrorOverflowError	Exception)groupall_gidgids      #platform/gsutil/gslib/tests/util.pyGetInvalidGidr$   @   s    ?ull?@G C 

	q 	Sq 

$ 1
221 @"   
 s   A<B B	BBc                      [        5       n [        5       n[        U5      S:X  a  U $ [        U5       Vs/ s H  o"U :w  d  M
  UPM     snS   $ s  snf )Nr   r   )GetPrimaryGidGetUserGroupslenlist)primary_giduser_groupsgs      r#   GetNonPrimaryGidr-   \   sN      /K/K
;1K(=(!,<A(=a@@=s   	A Ac                  ,    [         R                  " 5       $ N)osgetgid     r#   r&   r&   i   s    99;r3   c                      [        [        5       /[        R                  " 5        V s/ s H)  n [	        5       U R
                  ;   d  M  U R                  PM+     sn -   5      $ s  sn f r/   )setr&   r   r   	USER_NAMEgr_memr   )r,   s    r#   r'   r'   l   sL     "%,,.L.QIK1884K.LM N NLs   A 
A 
   c                  J    [         R                  " [        5      R                  $ r/   )pwdgetpwuidUSER_IDpw_namer2   r3   r#   <lambda>r>   s   s    #,,w"7"?"?r3   c                      [        [        R                  " 5        V s/ s H  o R                  PM     sn 5      S   S-   $ s  sn f )Nr   )r   r:   getpwallpw_uid)users    r#   r>   r>   x   s-    fclln=ndkkn=>rBQF=s   ?c                      [        5       $ r/   )r$   r2   r3   r#   r>   r>   }       MOr3   c                      [        5       $ r/   )r'   r2   r3   r#   r>   r>      rE   r3   Fc                    U(       a  SnO![         R                  R                  SSS5      nSSR                  U5      SS/n[        R
                  SSS[        5       -   /U-   U -   n[        5       (       d  [        [        R                  5      /U-   nOUnU Vs/ s H  n[        R                  " U5      PM     sn$ s  snf )	zFAdds config options to a list of strings defining a gsutil subcommand.FGSUtiluse_gcloud_storagez-ozGSUtil:use_gcloud_storage={}z#GSUtil:hidden_shim_mode=no_fallbackz--testexceptiontraceszGSUtil:default_project_id=)botoconfiggetboolformatgslibGSUTIL_PATHr   InvokedFromParFilestrsys
executablesix
ensure_str)raw_commandforce_gsutilrI   gcloud_storage_settinggsutil_command#gsutil_command_with_executable_pathparts          r#   GetGsutilCommandr\      s     ,,X7K-24 $++,>?
+	 0$"%6%88   ++. 
		+.s~~+>*?.*P'*8'+N	O+N4#..
+N	OO	Os    B<c           	         [         R                  R                  5       nU(       a  UR                  U5        [	        5       n[
        R                  " U5       H2  u  pE[
        R                  " U5      U[
        R                  " U5      '   M4     [        [         S5      (       a  [         R                  OSn[        R                  " U [        R                  [        R                  [        R                  UUS9$ )zCReturns a subprocess.Popen object for for running a gsutil command.setsidN)stdoutstderrstdinenv
preexec_fn)r0   environcopyupdatedictrT   	iteritemsrU   hasattrr^   
subprocessPopenPIPE)cmdenv_varsrb   envstrkvrc   s          r#   GetGsutilSubprocessrr      s    


#JJx6&mmC da #q 1F3>>! ! $B11ryyt*			#!+!+ *$%/
1 1r3   c           
        ^  UbU  [         R                  (       a+  [        U[        5      (       d  UR	                  [
        5      nOUR	                  [
        5      nSU0nU 4S jn[         R                  (       a  SUS'   O'[        R                  " SU5      nUR                  5         T R                  " S0 UD6n[         R                  (       d  WR                  5          U Vs/ s H  n[         R                  " U5      PM     nnU$ s  snf ! [         aE    U Vs/ s H.  n[         R                  " U[        R                  " S5      5      PM0     Os  snf nn U$ f = f)Ninputc                     > [         R                  " [         R                  " T R                  5      [        R
                  5        g r/   )r0   killpggetpgidpidsignalSIGKILL)processs   r#   Kill$CommunicateWithTimeout.<locals>.Kill   s"    IIbjj%v~~6r3   ih  timeoutFr2   )rT   PY3
isinstancebytesencoder   	threadingTimerstartcommunicatecancelensure_textUnicodeDecodeErrorlocalegetpreferredencoding)r{   ra   comm_kwargsr|   timerc_outoutputs   `      r#   CommunicateWithTimeoutr      s   

wwu%%T"ll4 e% +7 	WW K	OOC&E	KKM



,
,%		LLN3895S__V$5E9 
, :	  F 	 ; ;E BCE E
 
,s0   D " DD D E5EEEs,   iMSM9eeXliDZHSBJZO71R98tfeW/+87VXTpk5chGd6Y=s,   4TSaQ3S4U+5oxAbByA7HgIigD51zfzGed/c03Ts2TXc=s,   HO4Q2X28N/6SmuAJ1v1CTuJjf5emQcXf7YriKzT1gj0=s,   U6zIErjZCK/IpIeDS0pJrDayqlZurY8M9dvPJU0SXI8=s,   MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTI=s   dnF5x6K/8ZZRzpfSlMMM+w==s   barzN7UdGUp1E+RbVvZSTy1R8g==zCrcTMQ==s   bar2zIk4lOfUiA+szcorNIotEMg==zQScXtg==s   bar3z9iW6smjfu9hm0A//VQTQfw==zs0yUtQ==s   bar4zkPCx6uZiUOU7W6E+cDCZFg==zZ4bwXg==s   bar5z758XbXQOVkp8fTKMm83NXA==zle1zXQ==Tz&/tmp/.boto.parallel_upload_test_configzPThis sync will orphan file(s), please fix their permissions before trying again.z$Mode for %s won't allow read access.z+GID for %s doesn't exist on current system.z+UID for %s doesn't exist on current system.z,Insufficient access with uid/gid/mode for %sc                   0    \ rS rSrSrSrSrSrSrSr	Sr
S	rg
)KmsTestingResourcesi  zCConstants for KMS resource names to be used in integration testing.zus-central1z$keyring-for-gsutil-integration-testsz key-for-gsutil-integration-testsz!key-for-gsutil-integration-tests2zkey-for-gsutil-no-authz-cryptokey-for-gsutil-integration-tests-%d%d%dr2   N)__name__
__module____qualname____firstlineno____doc__KEYRING_LOCATIONKEYRING_NAMECONSTANT_KEY_NAMECONSTANT_KEY_NAME2"CONSTANT_KEY_NAME_DO_NOT_AUTHORIZEMUTABLE_KEY_NAME_TEMPLATE__static_attributes__r2   r3   r#   r   r     s2    K" 8, 9: (@$ Nr3   r   c                    [        [        R                  " 5       5      nUR                  [	        S5      [
        R                  [
        R                  S9nUR                  X 5      n[        SSSU/SS9n[        U5      n[        U5        U$ )zAEnsures test keys exist and that the service agent is authorized.N)locationkms	authorizez-kT)rW   )r
   logging	getLoggerCreateKeyRingr   r   r   r   CreateCryptoKeyr\   rr   r   )key_namekms_apikeyring_fully_qualified_namekey_fully_qualified_namerm   r{   s         r#   "AuthorizeProjectToUseTestingKmsKeyr   )  s     7$$&''!(!6!6&&"33 "7 "5
 %44".%d4LM&*	,#$'!	!!r3   c                 F    [         R                  " U[        U 5      -  5      $ )zBuilds a regex to match a file name for a file that would be orphaned.

Args:
  obj: Object uri.
  err_str: The error string to search for.

Returns:
  A regex that will match the file name and with the error text for a file
  that would be orphaned.
)recompileObjectToURI)objerr_strs     r#   BuildErrorRegexr   <  s     
Gk#..	//r3   c                 h   ^  [        U 4S jUR                  5       R                  S5       5       5      $ )a  Returns set of object name tails.

Tails can be compared between source and dest, past the point at which the
command was done. For example if test ran {cp,mv,rsync}
gs://bucket1/dir gs://bucket2/dir2, the tails for listings from bucket1
would start after "dir", while the tails for listings from bucket2 would
start after "dir2".

Args:
  start_point: The target of the cp command, e.g., for the above command it
               would be gs://bucket1/dir for the bucket1 listing results and
               gs://bucket2/dir2 for the bucket2 listing results.
  listing: The listing over which to compute tail.

Returns:
  Object name tails.
c              3   >   >#    U  H  o[        T5      S  v   M     g 7fr/   )r(   ).0lstart_points     r#   	<genexpr>TailSet.<locals>.<genexpr>\  s     G+Fas; !+Fs   
)r5   stripsplit)r   listings   ` r#   TailSetr   J  s'    $ 
G7==?+@+@+FG	GGr3   Credentialsaws_access_key_idaws_secret_access_keygs_service_key_file z.p12gs_hostgs_portrH   
prefer_apijsonXMLc                 J    [         (       a  [        R                  " U 5      $ S $ )Nc                     U $ r/   r2   funcs    r#   r>   !SkipForP12Creds.<locals>.<lambda>r      r3   )HAS_P12_CREDSunittestskipreasons    r#   SkipForP12Credsr   n  s    ]==  r3   c                  \    S n [         (       d	   SS Kn U S L$ U S L$ ! [         a     U S L$ f = f)Nr   )r   argcompleteImportError)r   s    r#   _ArgcompleteAvailabler   t  sJ    +	 
D	  D	    
	D	  s    
++c                 \   U R                  SS5      n [        [        R                  R	                  U 5      5      n[
        R                  " US   5      US'   US   R                  S5      (       a  US   SS US'   [        R                  R                  U5      nUR                  SS5      nU$ )zNormalizes the path component of a URI.

Args:
  uri: URI to normalize.

Returns:
  Normalized URI.

Examples:
  gs://foo//bar -> gs://foo/bar
  gs://foo/./bar -> gs://foo/bar
zgs://zfile://   z//r   N)	replacer)   r   parseurlparse	posixpathnormpath
startswith
urlunparse)uriparsedunparseds      r#   _NormalizeURIr     s    & 	GY'#%%c*+&  +&)AY$q	!"F1I\\$$V,(i1(	/r3   c                     U R                   (       d"  U R                  (       d  U R                  S:X  a  gU R                   =(       d    U R                  $ )zReturns a the generation for a StorageUri.

Args:
  uri: boto.storage_uri.StorageURI object to get the URI from.

Returns:
  Generation string for the URI.
s3null)
generation
version_idscheme)r   s    r#   GenerationFromURIr     s3     ..CNN
zzT		)3>>)r3   c                    [        U 5      (       aW  SR                  [        R                  R	                  [        R                  R
                  " U R                  /UQ76 5      5      $ [        U [        R                  5      (       a0  SR                  [        R                  R
                  " U /UQ76 5      $ [        R                  " U R                  5      nU(       aD  U Vs/ s H  n[        R                  " U5      PM     nn[        SR                  U/U-   5      5      nUR                  S5      (       a  USS nU$ s  snf )a  Returns the storage URI string for a given StorageUri or file object.

Args:
  obj: The object to get the URI from. Can be a file object, a subclass of
       boto.storage_uri.StorageURI, or a string. If a string, it is assumed to
       be a local on-disk path.
  *suffixes: Suffixes to append. For example, ObjectToUri(bucketuri, 'foo')
             would return the URI for a key name 'foo' inside the given
             bucket.

Returns:
  Storage URI string.
z	file://{}/Nr@   )is_filerM   r0   pathabspathjoinnamer   rT   string_typesr   r   r   endswith)r   suffixesr   suffixsuffixes_lists        r#   r   r     s     S\\bggoobggll388 /F<D/F G H HS%%&&bggll3::;; #;CD8S__V,8MD
#!67
8C 	\\#
cr(C	* Es   " D?c                   (   ^  \ rS rSrU 4S jrSrU =r$ )GSMockConnectioni  c                 F   > SUS'   SU l         [        [        U ]  " U0 UD6  g )Ngsproviderr   )debugsuperr   __init__)selfargskwargs	__class__s      r#   r   GSMockConnection.__init__  s)    F:DJ	
D*D;F;r3   )r   )r   r   r   r   r   r   __classcell__)r  s   @r#   r   r     s    < <r3   r   c                   d    \ rS rSrSS jrSS jrSS jrSS jrSS jrSS jr	SS	 jr
SS
 jrSrg)GSMockBucketStorageUrii  Nc                     [         $ r/   )mock_connection)r  access_key_idsecret_access_keys      r#   connectGSMockBucketStorageUri.connect  s    r3   c                 "    U R                  5       $ )z:Dummy implementation to allow parallel uploads with tests.)new_key)r  
componentsheaderss      r#   composeGSMockBucketStorageUri.compose  s    <<>r3   c                     g)NUSr2   r  r  s     r#   get_location#GSMockBucketStorageUri.get_location      r3   c                 R    [         R                  R                  R                  5       $ r/   )rJ   r   corsCorsr  s     r#   get_corsGSMockBucketStorageUri.get_cors  s    77<<r3   c                 R    [         R                  R                  R                  5       $ r/   )rJ   r   encryptionconfigEncryptionConfigr  s     r#   get_encryption_config,GSMockBucketStorageUri.get_encryption_config  s    77##4466r3   c                     g r/   r2   r  s     r#   get_lifecycle_config+GSMockBucketStorageUri.get_lifecycle_config  r  r3   c                     g r/   r2   r  s     r#   get_website_config)GSMockBucketStorageUri.get_website_config  r  r3   c                     g r/   r2   r  s     r#   get_versioning_config,GSMockBucketStorageUri.get_versioning_config  r  r3   r2   )NNr/   )r   r   r   r   r  r  r  r  r#  r&  r)  r,  r   r2   r3   r#   r  r    s*    7r3   r  TestRemoveSectionc                    [         R                  R                  XS5      n[         R                  R                  U 5      (       d7  UR	                  U [
        S45        [         R                  R                  U 5        UR	                  XU45        Uc   [         R                  R                  X5        g[         R                  R                  XU5        g)aw  Sets boto configuration temporarily for testing.

SetBotoConfigForTest should be called by tests instead of this function.
This will ensure that the configuration is reverted to its original setting
using _RevertBotoConfig.

Args:
  section: Boto config section to set
  name: Boto config name to set
  value: Value to set
  revert_list: List for tracking configs to revert.
N)	rJ   rK   gethas_sectionappendTEST_BOTO_REMOVE_SECTIONadd_sectionremove_optionr5   )sectionr   valuerevert_list
prev_values        r#   _SetBotoConfigr:    s     {{wd3*		 	 	)	)!94@AKKG$gZ01
]KKg,KKOOG5)r3   c                 2   / nU  Hg  u  p#nUc>  U[         :X  a  UR                  U5        M&  [        R                  R	                  X#5        MG  [        R                  R                  X#U5        Mi     U H"  n[        R                  R                  U5        M$     g)zReverts boto config modifications made by _SetBotoConfig.

Args:
  revert_list: List of boto config modifications created by calls to
               _SetBotoConfig.
N)r3  r2  rJ   rK   r5  r5   remove_section)r8  sections_to_remover6  r   r7  s        r#   _RevertBotoConfigr>    sw     )gU}	)	)!!'*!!'0
kkoogU+ * $gKKw' $r3   c                 F   ^  [         R                  " T 5      U 4S j5       nU$ )aP  Decorator for tests that perform file to object transfers, or vice versa.

This forces the test to run once normally, and again with special boto
config settings that will ensure that the test follows the parallel composite
upload and/or sliced object download code paths.

Args:
  func: Function to wrap.

Returns:
  Wrapped function.
c                     > T" U 0 UD6  [         (       d/  [        5       (       a  [        / SQ5         T" U 0 UD6  S S S 5        g g g ! , (       d  f       g = f)N))rH   #parallel_composite_upload_threshold1)rH    sliced_object_download_thresholdrB  )rH   %sliced_object_download_max_components3)rH   check_hashesalways)RUN_S3_TESTSr   SetBotoConfigForTest)r  r  r   s     r#   Wrapper.SequentialAndParallelTransfer.<locals>.Wrapper1  sX     	$&<022 ! 	 	df	 	 3<	 	s   	A
A)	functoolswraps)r   rJ  s   ` r#   SequentialAndParallelTransferrN  #  s'     ??4  
.r3   c                 b    0 nU  H&  nUS   US   US   pTnX1;  a  0 X'   Uc  M  XQU   U'   M(     U$ )a  Converts the input config list to a dict that is easy to write to a file.

This is used to reset the boto config contents for a test instead of
preserving the existing values.

Args:
  boto_config_list: list of tuples of:
      (boto config section to set, boto config name to set, value to set)
      If value to set is None, no entry is created.

Returns:
  Dictionary of {section: {keys: values}} for writing to the file.
r   r   r   r2   )boto_config_listsectionsconfig_entryr6  keyr7  s         r#   _SectionDictFromConfigListrT  C  sR     (&l'?LO\!_%Gh$w ' 
/r3   c           	      &   [        US5       n[        R                  " U 5       HQ  u  p4UR                  SU-  5        [        R                  " U5       H  u  pVUR                  U< SU< S35        M      MS     SSS5        g! , (       d  f       g= f)zFWrites a section dict from _SectionDictFromConfigList to tmp_filename.wz[%s]
z = r   N)openrT   rh   write)section_dicttmp_filenametmp_filer6  key_value_pairsrS  r7  s          r#   _WriteSectionDictToFiler]  \  sh    L#($'MM,$? nnX'(o6*#c512 7 %@ s   A,B
Bc               #   `   #    [        S/5         Sv   SSS5        g! , (       d  f       g= f7f)zHSets a dummy project in boto config for the duration of a 'with' clause.)rH   default_project_id
dummy_projN)rI  r2   r3   r#   SetDummyProjectForUnitTestra  e  s"     
 KLM	 NMMs   .	.
+.c              #   8  #    / nSn [         R                  " SS9u  pC[        R                  " U5        U(       a  U  H[  nUS   n[        R
                  (       a*  [        U[        5      (       a  UR                  [        5      n[        US   US   UU5        M]     [        US5       n[        R                  R                  U5        SSS5        O[        [!        U 5      U5        [#        U5         Sv   SSS5        [%        U5        U(       a   [        R&                  " U5        gg! , (       d  f       NQ= f! , (       d  f       NJ= f! [(         a     gf = f! [%        U5        U(       a(   [        R&                  " U5        f ! [(         a     f f = ff = f7f)a+  Sets the input list of boto configs for the duration of a 'with' clause.

This preserves any existing boto configuration unless it is overwritten in
the provided boto_config_list.

Args:
  boto_config_list: list of tuples of:
      (boto config section to set, boto config name to set, value to set)
  use_existing_config: If True, apply boto_config_list to the existing
      configuration, preserving any original values unless they are
      overwritten. Otherwise, apply boto_config_list to a blank configuration.

Yields:
  Once after config is set.
Nzgsutil-temp-cfg)prefixr   r   r   rV  )tempfilemkstempr0   closerT   r   r   r   decoder   r:  rW  rJ   rK   rX  r]  rT  _SetBotoConfigFileForTestr>  removeOSError)rP  use_existing_configrevert_configsrZ  tmp_fdboto_config
boto_valuer[  s           r#   rI  rI  n  sV    " .,#++3DEFHHV)+ ^
77
E**#**40J{1~{1~z%	' * c"h(# #" 89IJ*, 
#<	0 
1 n%
		,  #" 
1	0  	 n%
		,  s   FB E ( D))E 1D:6E >FE 'F)
D73E :
EE 
EFEFF/FF
FFFFFc              #      ^#    U4S jn0 mU  H%  n[         R                  R                  U5      TU'   M'      U" U 5        Sv   U" T5        g! U" T5        f = f7f)z0Sets OS environment variables for a single test.c                   > [         R                  " U 5       Hf  u  p[        R                  R	                  U5      TU'   Ub  U[        R                  U'   M?  U[        R                  ;   d  MU  [        R                  U	 Mh     g r/   )rT   rh   r0   rd   r0  )dict_to_applyrp   rq   
old_valuess      r#   _ApplyDictToEnvironment6SetEnvironmentForTest.<locals>._ApplyDictToEnvironment  sV    m,jjnnQ'jm	


1

?JJqM -r3   N)r0   rd   r0  )env_variable_dictrt  rp   rs  s      @r#   SetEnvironmentForTestrw    sT      *aJJNN1%JqM (-.	J'J's   4AA 	A
AAc              #     #     [         R                  S   nSnU [         R                  S'    Sv   U(       a  W[         R                  S'   g[         R                  R                  SS5        g! [         a    Sn Nbf = f! U(       a  W[         R                  S'   f [         R                  R                  SS5        f = f7f)au  Sets a given file as the boto config file for a single test.

This function applies only the configuration in boto_config_path and will
ignore existing configuration. It should not be called directly by tests;
instead, use SetBotoConfigForTest.

Args:
  boto_config_path: Path to config file to use.

Yields:
  When configuration has been applied, and again when reverted.
BOTO_CONFIGTFN)r0   rd   r   pop)boto_config_pathold_boto_config_env_variableboto_config_was_sets      r#   rh  rh    s      #%::m#<  /"**]*	 ">bjjjjnn]D) 
    ">bjjjjnn]D)s=   B?A- B?A? <B?-A<9B?;A<<B??=B<<B?c                     [         R                  " S5      n / n[        R                  " [        R
                  5       H@  u  p#nU R                  U5      nU(       d  M   UR                  UR                  S5      5        MB     U$ )z?Returns a list of the names of the test modules in gslib.tests.z^test_(?P<name>.*)$r   )	r   r   pkgutiliter_modulesgslib_tests__path__matchr2  r    )matchernames_modnamems        r#   GetTestNamesr    sd    JJ-.'
%++K,@,@Ama!gAqll1776?# B 
,r3   c                     [         R                  (       a  [        U [        5      $ [        U [        R
                  5      $ r/   )rT   PY2r   fileioIOBase)r   s    r#   r   r     s'    WWc4  	C	##r3   c                     [        U [        R                  [        R                  45      (       a   U R	                  SS5      R                  5       $ [        SR                  [        U 5      5      5      e)a  Returns a copy of the given name with any invalid characters replaced.

Args:
  name Union[str, unicode, bytes]: The bucket name to transform into a valid name.

Returns:
  Union[str, unicode, bytes] The version of the bucket name containing only
    valid characters.
r  -z*Unable to format name. Incorrect Type: {0})	r   rT   	text_typebinary_typer   lower	TypeErrorrM   type)r   s    r#   MakeBucketNameValidr    sZ     s}}coo677<<S!''))
@GGT
  r3   c              #   B  #    Sn [         R                  " 5       nU (       a  [         R                  " U 5         Sv   U (       a  U(       a  [         R                  " U5        ggg! [         a     NUf = f! U (       a  U(       a  [         R                  " U5        f f f = f7f)a  Changes the working directory for the duration of a 'with' call.

Args:
  new_working_directory: The directory to switch to before executing wrapped
    code. A None value indicates that no switching is necessary.

Yields:
  Once after working directory has been changed.
N)r0   getcwdrj  chdir)new_working_directoryprev_working_directorys     r#   WorkingDirectoryr    s       	YY[
 HH"#'	!7hh%& "8 
 		 !7hh%& "8s=   BA$ BA4 'B$
A1.B0A11B4(BBc                  x    [        5       R                  SS 5      n U (       d  gSU R                  R                  ;   $ )N
__loader__F	zipimport)globalsr0  r  r   )loaders    r#   rP   rP     s1    9==t,&		((33	33r3   c                 R    [        5       (       a  [        R                  " U 5      $ S $ )Nc                     U $ r/   r2   r   s    r#   r>    SkipForParFile.<locals>.<lambda>  r   r3   )rP   r   r   r   s    r#   SkipForParFiler    s     ==  r3   c                   $    \ rS rSrSrS rS rSrg)HaltingCopyCallbackHandleri#  zFTest callback handler for intentionally stopping a resumable transfer.c                     Xl         X l        g r/   )
_is_upload_halt_at_byte)r  	is_uploadhalt_at_bytes      r#   r   #HaltingCopyCallbackHandler.__init__&  s    O%r3   c           
          XR                   :  ao  [        R                  R                  SU R                   < S[	        U5      < S[	        U5      < S35        U R
                  (       a  [        S5      e[        S5      eg)a0  Forcibly exits if the transfer has passed the halting point.

Note that this function is only called when the conditions in
gslib.progress_callback.ProgressCallbackWithTimeout.Progress are met, so
self._halt_at_byte is only precise if it's divisible by
gslib.progress_callback._START_BYTES_PER_CALLBACK.
zHalting transfer after byte z. r   z transferred.
zArtifically halting upload.Artifically halting download.N)r  rR   r`   rX  r   r  r   r   )r  total_bytes_transferred
total_sizes      r#   callHaltingCopyCallbackHandler.call+  sg     "4"44	jj01HIZ(*+ 
&'DEE()HII 5r3   )r  r  Nr   r   r   r   r   r   r  r   r2   r3   r#   r  r  #  s    N&
Jr3   r  c                   $    \ rS rSrSrS rS rSrg)#HaltOneComponentCopyCallbackHandleri>  z=Test callback handler for stopping part of a sliced download.c                     S U l         Xl        g r/   )_last_progress_byter  )r  r  s     r#   r   ,HaltOneComponentCopyCallbackHandler.__init__A  s    #D%r3   c                     U R                   bN  U R                   U R                  s=:  a  U:  a-  O  O*[        R                  R	                  S5        [        S5      eXl         g)zCForcibly exits if the passed the halting point since the last call.NzHalting transfer.
r  )r  r  rR   r`   rX  r   )r  current_progress_bytetotal_size_unuseds      r#   r  (HaltOneComponentCopyCallbackHandler.callG  sM      ,  4#5#5M8MM	jj./&'FGG4r3   )r  r  Nr  r2   r3   r#   r  r  >  s    E&5r3   r  c                   "    \ rS rSrSrSS jrSrg)
TestParamsiP  a_  Allows easier organization of test parameters.

This class allows grouping of test parameters, which include args and kwargs
to be used, as well as the expected result based on those arguments.

For example, to test an Add function, one might do:

params = TestParams(args=(1, 2, 3), expected=6)
self.assertEqual(Add(*(params.args)), params.expected)
Nc                     Uc
  [        5       OUU l        Uc
  [        5       OUU l        X0l        [        U[         [        45      (       d  [        S5      e[        U R                  [        5      (       d  [        S5      eg )Nz'TestParam args must be a tuple or list.z TestParam kwargs must be a dict.)tupler  rg   r  expectedr   r)   r  )r  r  r  r  s       r#   r   TestParams.__init__\  sd    <TDI"N$&DKMdUDM**?@@dkk4((899 )r3   )r  r  r  )NNN)r   r   r   r   r   r   r   r2   r3   r#   r  r  P  s    	:r3   r  c                   $    \ rS rSrSrS rS rSrg)CaptureStdoutig  zContext manager.

For example, this function has the lines printed by the function call
stored as a list in output:

with CaptureStdout() as output:
  function(input_to_function)
c                 d    [         R                  U l        [        5       =[         l        U l        U $ r/   )rR   r_   _stdoutr	   	_stringio)r  s    r#   	__enter__CaptureStdout.__enter__q  s"    ::DL"++-CJKr3   c                     U R                  U R                  R                  5       R                  5       5        U ?U R                  [
        l        g r/   )extendr  getvalue
splitlinesr  rR   r_   )r  r  s     r#   __exit__CaptureStdout.__exit__v  s4    KK'')4467CJr3   )r  r  N)r   r   r   r   r   r  r  r   r2   r3   r#   r  r  g  s    
r3   r  )Fr/   )T)
__future__r   r   r   r   
contextlibr   rL  r   r   r0   r  r   r   r  ry   rj   rR   rd  r   r   rT   	six.movesr   r	   rJ   crcmodrN   gslib.kms_apir
   gslib.project_idr   mock_storage_servicegslib.cloud_apir   r   gslib.lazy_wrapperr   gslib.teststestsr  gslib.utilsr   gslib.utils.boto_utilr   r   gslib.utils.constantsr   gslib.utils.encryption_helperr   gslib.utils.system_utilr   gslib.utils.unit_utilr   r   r:   r$   r-   r&   r'   InitializeDefaultModeintSYSTEM_POSIX_MODEDEFAULT_MODEgetuidr<   r6   INVALID_UIDINVALID_GIDUSER_GROUPSr\   rr   r   TEST_ENCRYPTION_KEY1TEST_ENCRYPTION_KEY1_SHA256_B64TEST_ENCRYPTION_KEY2TEST_ENCRYPTION_KEY2_SHA256_B64TEST_ENCRYPTION_KEY3TEST_ENCRYPTION_KEY3_SHA256_B64TEST_ENCRYPTION_KEY4TEST_ENCRYPTION_KEY4_SHA256_B64TEST_ENCRYPTION_KEY_S3TEST_ENCRYPTION_KEY_S3_MD5TEST_ENCRYPTION_CONTENT1TEST_ENCRYPTION_CONTENT1_MD5TEST_ENCRYPTION_CONTENT1_CRC32CTEST_ENCRYPTION_CONTENT2TEST_ENCRYPTION_CONTENT2_MD5TEST_ENCRYPTION_CONTENT2_CRC32CTEST_ENCRYPTION_CONTENT3TEST_ENCRYPTION_CONTENT3_MD5TEST_ENCRYPTION_CONTENT3_CRC32CTEST_ENCRYPTION_CONTENT4TEST_ENCRYPTION_CONTENT4_MD5TEST_ENCRYPTION_CONTENT4_CRC32CTEST_ENCRYPTION_CONTENT5TEST_ENCRYPTION_CONTENT5_MD5TEST_ENCRYPTION_CONTENT5_CRC32CRUN_INTEGRATION_TESTSRUN_UNIT_TESTSrH  USE_MULTIREGIONAL_BUCKETS%PARALLEL_COMPOSITE_UPLOAD_TEST_CONFIGORPHANED_FILEPOSIX_MODE_ERRORPOSIX_GID_ERRORPOSIX_UID_ERRORPOSIX_INSUFFICIENT_ACCESS_ERRORobjectr   r   r   r   r   rK   r0  HAS_S3_CREDSr   r   HAS_NON_DEFAULT_GS_HOSTHAS_GS_HOSTHAS_GS_PORTupperUSING_JSON_APIr   r   ARGCOMPLETE_AVAILABLEr   r   r   MockConnectionr   r
  MockBucketStorageUrir  r3  r:  r>  rN  rT  r]  ra  rI  rw  rh  r  r   r  r  rP   r  r  r  r  r)   r  r2   r3   r#   <module>r     s    ' %  ' %    	   	 	   
    
        .  6 4 * ! " N & M . 3 38AN ""$Z1115,IIK'?@) FH+ 34+ 34+P>1, L G "E#  G "E#  G "E#  G "E#  I 8 ! 9 ", " 9 ", " 9 ", " 9 ", " 9 ",   ! (P %( ; @@"P N& N. !22"&0H* /BDI N/FM  /DbIRRSYZ02 kkoomY=TIkkoomY=TI<!')).E:! ./ <*<<+:: < #$1FF 8 / *0((@23 
 
 + +\ ( (, * *>$& ' '64J J65& 5$: :.D r3   