
    h                     ~   S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKJr  SSKrSSKJr  SS	KJr  SS
KJr  SSKJr  SSKrSSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJ r   SSK!J"r"  SSK!J#r#  SSK$r$SSK%J&r&  \
RN                  (       a  \(r)\RT                  " SSS5      r+\RT                  " SSS5      r,Sq-Sq.S r/S r0S r1S r2S r3S r4S  r5S! r6S" r7S# r8S$ r9S% r:S& r;S' r<S( r=\$R|                  4S) jr?S* r@S+ rAS, rBS- rCS. rDS/ rES0 rFS1 rGS2 rHS3 rIS7S4 jrJS5 rKS6 rLg)8zShared utility structures and methods that require importing boto.

This module also imports httplib2 (as it is Boto's http transport and closely
tied to some of Boto's core functionality) and oauth2client.
    )absolute_import)print_function)division)unicode_literalsN)config)NoAuthHandlerFound)GSConnection)Provider)BotoConfigLocations)context_config)CommandException)system_util)DEFAULT_GCS_JSON_API_VERSION)DEFAULT_GSUTIL_STATE_DIR)SSL_TIMEOUT_SEC)UTF8)HumanReadableToBytes)ONE_MIB)
HAS_CRYPTOBoto	is_secureThttps_validate_certificatesc                     [         R                  R                  SSS5      n U S:X  a  gU (       Gd  [        (       Gd
  [        R
                  R                  [        R
                  R                  [        R                  SS5      5      q[        R
                  R                  [        5      (       d  [        R                  " SS5      nU(       d  [        S	5      e[        R                  " U5      n[         R"                  " S
SS9u  p#[        R$                  " US5      nUR'                  U5        UR)                  5         Uq[*        q[        n U $ )zConfigures and returns the CA Certificates file.

If one is already configured, use it. Otherwise, use the cert roots
distributed with gsutil.

Returns:
  string filename of the certs file to use.
r   ca_certificates_fileNsystemdatazcacerts.txtgslibzdata/cacerts.txtzACertificates file not found. Please reinstall gsutil from scratchz.txtzgsutil-cacerts)suffixprefixw)botor   getconfigured_certs_fileospathabspathjoinr   	GSLIB_DIRexistspkgutilget_datar   six
ensure_strtempfilemkstempfdopenwriteclosetemp_certs_file)
certs_file
certs_datafdfnamefs        (platform/gsutil/gslib/utils/boto_util.pyConfigureCertsFiler:   B   s     {{v'=tD*
 8	   ggoo
'',,u
>@WW^^122 %%g/AB
  "A B B^^J/
$$F;KL	IIb#	
		 /&J	    c                  j   [        5       (       d  [        R                  " SS5      (       a  [        (       dv  [        R
                  " 5       (       a.  [        SR                  [        R                  " S5      5      5      e[        SR                  [        R                  " S5      5      5      eSSK
Jn   gg)	zASets up no-op auth handler if no boto credentials are configured.Credentialsgs_service_client_id
aI  Your gsutil is configured with an OAuth2 service account, but you do not have PyOpenSSL or PyCrypto 2.6 or later installed. Service account authentication requires one of these libraries; please reactivate your service account via the gcloud auth command and ensure any gcloud packages necessary for service accounts are present.a(  Your gsutil is configured with an OAuth2 service account, but you do not have PyOpenSSL or PyCrypto 2.6 or later installed. Service account authentication requires one of these libraries; please install either of them to proceed, or configure a different type of credentials with "gsutil config".r   no_op_auth_pluginN)HasConfiguredCredentialsr   
has_optionr   r   InvokedViaCloudSdkr   r'   textwrapwrapr   rA   r@   s    r9   ConfigureNoOpAuthIfNeededrG   l   s    	!	#	#-)?@@J		'	'	)	)tyyMM01 2 3 	3 tyyMMFG H I 	I */ 
$r;   c                      [         $ N)r#    r;   r9   GetCertsFilerK      s    	r;   c                  *    [         (       a  [         /$ / $ )zHReturns a list of temp files to delete (if possible) when program exits.)r3   rJ   r;   r9   GetCleanupFilesrM      s    -o/	525r;   c                     [         n S[        R                  ;   a"  U R                  [        R                  S   5        [        R                  R                  [        R                  R                  S5      SS5      n[        R                  R                  U5      (       a  U R                  U5        / nU  H*  n [        US5         UR                  U5        SSS5        M,     U$ ! , (       d  f       M?  = f! [         a     MO  f = f)zFReturns a list of the path(s) to the boto config file(s) to be loaded.AWS_CREDENTIAL_FILE~z.awscredentialsrN)
r   r$   environappendr%   r'   
expanduserisfileopenIOError)potential_config_pathsaws_cred_filereadable_config_pathsr%   s       r9   GetConfigFilePathsr\      s    .
 bjj(!!"**-B"CD
 '',,rww11#6N-WW^^M""!!-0 $dc?$$T* ? % 
	 ? 
s0   8C6C$C6$
C3	.C63C66
DDc                  Z    [        5       n [        U 5      S:X  a  U R                  S5        U $ )zGLike GetConfigFilePaths but returns a not-found message if paths empty.r   zNo config found)r\   lenrT   )r[   s    r9   GetFriendlyConfigFilePathsr_      s.    ,.	1$  !23	r;   c                  j    [         R                  " SS[        5      n [        R                  " U 5        U $ )a  Returns the location of the directory for gsutil state files.

Certain operations, such as cross-process credential sharing and
resumable transfer tracking, need a known location for state files which
are created by gsutil as-needed.

This location should only be used for storing data that is required to be in
a static location.

Returns:
  Path to directory for gsutil static state files.
GSUtil	state_dir)r   r"   r   r   CreateDirIfNeeded)config_file_dirs    r9   GetGsutilStateDirre      s+     JJx6NO/0	r;   c                  R    [         R                  R                  [        5       S5      $ )N
credstore2r$   r%   r'   re   rJ   r;   r9   GetCredentialStoreFilenameri      s     
')<	88r;   c                  R    [         R                  R                  [        5       S5      $ )Ngcecredcacherh   rJ   r;   r9   GetGceCredentialCacheFilenamerl      s    	')>	::r;   c                  :    [         R                  " SS[        5      $ )Nra   json_api_version)r   r"   r   rJ   r;   r9   GetGcsJsonApiVersionro      s    	H02N	OOr;   c                      [         R                  " SS[        S5      5      n U S:X  a  [        S5      n U $ U [        S5      -  S:w  a  U [        S5      U [        S5      -  -
  -  n U $ )Nra   json_resumable_chunk_sizei  @r   i   )r   getintlong)
chunk_sizes    r9   GetJsonResumableChunkSizeru      so    }}X'B!"346*1_j!J 
 D$$)4
#zT*5E'FGHJ	r;   c                  R    [         R                  R                  [        5       S5      $ )Nz.last_software_update_checkrh   rJ   r;   r9   *GetLastCheckedForGsutilUpdateTimestampFilerw      s    	')+H	IIr;   c                      [        5       n S[        -  nX-   S-   SUS-
  S-  S-   -  -   n[        5       U-  nUS::  a  SnU$ )zGets the max concurrent transport compressed uploads allowed in parallel.

Returns:
  The max number of concurrent transport compressed uploads allowed in
  parallel without exceeding the max_upload_compression_buffer_size.
            i?  r   )ru   r   !GetMaxUploadCompressionBufferSize)upload_chunk_sizecompression_chunk_sizetotal_upload_sizemax_concurrent_uploadss       r9   !GetMaxConcurrentCompressedUploadsr      sk     01<(ABF014=BJD D=?-.q 	r;   c                  2    [         R                  " SSS5      $ )Nr   max_retry_delay    r   rr   rJ   r;   r9   GetMaxRetryDelayr      s    	v0"	55r;   c                  D    [        [        R                  " SSS5      5      $ )zEGet the max amount of memory compressed transport uploads may buffer.ra   "max_upload_compression_buffer_size2GiB)r   r   r"   rJ   r;   r9   r}   r}      s"    	jj?H
J Jr;   c           
         [         R                  " SSS5      nU[         R                  " SSS5      [         R                  " SS5      [         R                  " SSS5      [         R                  " SSS5      [         R                  " SS	U(       a  S
OS5      S.n[        U5      n[	        5       US'   [
        US'   U " SSU0UD6n[         R                  " SS5      (       + Ul        [        R                  " 5       nU(       aA  UR                  (       a0  UR                  UR                  UR                  SUR                  S9  U$ )zCreates and returns a new httplib2.Http instance.

Args:
  http_class: Optional custom Http class to use.
  **kwargs: Arguments to pass to http_class constructor.

Returns:
  An initialized httplib2.Http instance.
r   proxyN
proxy_typehttp
proxy_port
proxy_user
proxy_pass
proxy_rdnsT
proxy_hostr   r   r   r   r   ca_certstimeout
proxy_infor    )keycertdomainpasswordrJ   )r   r"   rr   SetProxyInforK   r   getbool"disable_ssl_certificate_validationr   get_context_configuse_client_certificateadd_certificateclient_cert_pathclient_cert_password)
http_classkwargsr   boto_proxy_configr   r   global_context_configs          r9   
GetNewHttpr     s    zz&'40* 
**V\6
2
--
-
**V\4
0
**V\4
0
**V\:44
H  -.* $~&%&		4z	4V	4$17+2- .-$) );;=4KK2CC3DD ""7"L"L  N
 
+r;   c                  2    [         R                  " SSS5      $ )Nr   num_retries   r   rJ   r;   r9   GetNumRetriesr   8  s    	v}b	11r;   c                  R    [         R                  R                  [        5       S5      $ )Nztab-completion-logsrh   rJ   r;   r9   GetTabCompletionLogFilenamer   <  s    	')+@	AAr;   c                      [         R                  R                  [        5       S5      n [        R
                  " U SS9  [         R                  R                  U S5      $ )Nztab-completioni  )modecache)r$   r%   r'   re   r   rc   )tab_completion_dirs    r9   GetTabCompletionCacheFilenamer   @  sC    ww||$5$79IJ 2?	('	22r;   c                     [         R                  " SS5      =(       a    [         R                  " SS5      n [         R                  " SS5      =(       a    [         R                  " SS5      n[         R                  " SS5      n[         R                  " SS5      n[         R                  " SS5      n[        =(       a5    [         R                  " SS	5      =(       a    [         R                  " SS
5      nU (       d#  U(       d  U(       d  U(       d  U(       d  U(       a  gSn [        R                  R                  [        R                  [         [        S5      S/S9nS[        [        USS5      SS5      :X  a  SnU$ ! [         a     U$ f = f)z1Determines if boto credential/config file exists.r=   gs_access_key_idgs_secret_access_keyaws_access_key_idaws_secret_access_keygs_oauth2_refresh_tokengs_external_account_file(gs_external_account_authorized_user_filer>   gs_service_key_fileTNgoogles3requested_capabilityNoOpAuth	__class____name__)r   rC   r   r!   authget_auth_handlerr	   DefaultHostr
   getattrr   )has_goog_credshas_amzn_credshas_oauth_credshas_external_creds*has_external_account_authorized_user_credshas_service_account_credsvalid_auth_handlers          r9   rB   rB   G  sx   %%m5GH M%%m5KL %%m5HI N%%m5LM &&}'@B/))-*DF060A0A?1A,  >'=>>'<= 
 /#50	33L4L4L4:4<X4FJN 4 Q W"K6    
 
 			s    AE< <
F
	F
c                  :    [         R                  " SSS 5      n U S L$ )Nra   rq   )r   r"   )chunk_size_defineds    r9   JsonResumableChunkSizeDefinedr   q  s!    zz(,GN	4	''r;   c                  P  ^^^^^^ SSK m[        R                  R                  mSUU4S jjn U [        R                  l        SSKmSSKm[        R                  R                  m[        R                  R                  mUUUU4S jnU[        R                  R                  l
        g)a  Apply gsutil-specific patches to Boto.

Here be dragons. Sorry.

Note that this method should not be used as a replacement for contributing
fixes to the upstream Boto library. However, the Boto library has historically
not been consistent about release cadence, so upstream fixes may not be
available immediately in a version which we can pin to. Also, some fixes may
only be applicable to gsutil. In such cases, patches should be applied to the
Boto library here (and removed if/when they are included in the upstream
repository and included in an official new release that we pull in). This
method should be invoked after all other Boto-related initialization has been
completed.
r   Nc                   > T" XS9nTR                   R                  TR                   R                  4n[        U Vs/ s H  oDU;  d  M
  UPM     snS S9[        U Vs/ s H  oDU;   d  M
  UPM     snS S9-   nU$ s  snf s  snf )Nr   c                     U R                   $ rI   r   	handler_ts    r9   <lambda>BMonkeyPatchBoto.<locals>._PatchedGetPluginMethod.<locals>.<lambda>  
    9#5#5r;   )r   c                     U R                   $ rI   r   r   s    r9   r   r     r   r;   )oauth2_pluginOAuth2ServiceAccountAuth
OAuth2Authsorted)clsr   handler_subclassesxml_oauth2_handlersrR   
new_resultgcs_oauth2_boto_pluginorig_get_plugin_methods         r9   _PatchedGetPluginMethod0MonkeyPatchBoto.<locals>._PatchedGetPluginMethod  s    /8 	,,EE,,779 	*K*17J.JQ*K5	

 	*G*13F.FQ*G5	
	
	   L
 Hs   	A=
A=	B
,B
c                 j  > [        U S5      (       a3  TR                  U R                  U R                  4U R                  5      nO'TR                  U R                  U R                  45      nSnU R
                  (       a  USU R
                  -  -  nOUS-  n[        R                  R                  U5        [        T	S5      (       GaY  [        T	SS5      (       GaF  T	R                  T	R                  5      nT	R                  Ul        U R
                  (       a  UR                  U R
                  5        U R                  (       a&  UR!                  U R                  U R"                  5        UR%                  XR                  S9U l        U R"                  U R&                  l        U R                  U R&                  l        UR                  U R&                  l        T	R                  U R&                  l        U R
                  U R&                  l        S U R&                  l        O@T	R%                  UU R"                  U R                  T	R                  U R
                  S	9U l        U R&                  R3                  5       nU R                  R5                  S
S5      S   nT" XE5      (       d  T" XTSU-  5      eg )Nr   zwrapping ssl socket; zCA certificate file=%szusing system provided SSL certs
SSLContextHAS_SNIF)server_hostname)keyfilecertfile	cert_reqsr   :r   z/remote hostname "%s" does not match certificate)hasattrcreate_connectionhostportr   r   r!   logdebugr   r   PROTOCOL_SSLv23CERT_REQUIREDverify_modeload_verify_locations	cert_fileload_cert_chainkey_filewrap_socketsockr   r   r   ssl_versionciphersgetpeercertsplit)
selfr   msgcontextr   hostnameInvalidCertificateExceptionValidateCertificateHostnamesocketssls
         r9   _PatchedConnectMethod.MonkeyPatchBoto.<locals>._PatchedConnectMethod  s    tY%%tyy$))&<dllKd%%tyy$))&<=d
!C}}	%55c	..cHHNN3sL!!gc9e&D&D s223g--g	%%dmm4	>%%dII%Fdi--dii>>dii#//dii!11dii==diidii//$*.--+/>>,/,=,=+/==	 " :di
 99  "DyysA&q)H&t66'
 "#$ $ 7r;   rI   )r   r!   plugin
get_pluginr
  r  https_connectionr  r	  CertValidatingHTTPSConnectionconnect)r   r  r  r	  r   r   r
  r  s     @@@@@@r9   MonkeyPatchBotor  v  s    2    ;;11 2 3$++ 
77  77 )$ )$Z  55=r;   c                    [         R                  R                  U 5      nU(       a$  U R                  5       R	                  S5      (       d0  [
        R                  " [
        R                  R                  SS5      $ U R                  5       R                  S5      S   nUR                  5       R	                  S5      (       d  US-   U-   n[
        R                  " XS9$ )zReads proxy info from the environment and converts to httplib2.ProxyInfo.

Args:
  proxy_env_var: Environment variable string to read, such as http_proxy or
     https_proxy.

Returns:
  httplib2.ProxyInfo constructed from the environment string.
r   Nr   _z://)method)r$   rS   r"   lower
startswithhttplib2	ProxyInfosocksPROXY_TYPE_HTTPr  proxy_info_from_url)proxy_env_var	proxy_urlproxy_protocols      r9   ProxyInfoFromEnvironmentVarr!    s     jjnn]+)	---/::6BBhnn<<dAFF &&(..s3A6.			%	%f	-	-&2I		%	%i	GGr;   c                  @    [         R                  " SSS[        -  5      $ )Nra   resumable_threshold   )r   rr   r   rJ   r;   r9   ResumableThresholdr%    s    	x!6G	DDr;   c           	         SSSSS.nUR                  U R                  S5      R                  5       US   5      nU R                  S5      nU R                  S5      nU R                  S	5      nU R                  S
5      n[        U R                  S5      5      n[        R                  " UUUUUUS9nUR
                  US   :X  d  SUl        UR                  (       a  UR                  (       da  S H[  n	U	[        R                  ;   d  M  [        R                  U	   (       d  M3  [        U	5      nU R                  S5      S:X  a  SUl          U$    U$ )zSets proxy info from boto and environment and converts to httplib2.ProxyInfo.

Args:
  dict: Values read from the .boto file

Returns:
  httplib2.ProxyInfo constructed from boto or environment variable string.
r|         )socks4socks5r   httpsr   r   r   r   r   r   r   r   F)
http_proxyhttps_proxyHTTPS_PROXYNT)r"   r  boolr  r  r   r   r   r   r$   rS   r!  )
r   proxy_type_specr   r   r   r   r   r   r   r  s
             r9   r   r     sE     !Aq1E/ ""L)//1?63JL* $$\2* $$\2* $$\2* $$\2*%)),78* !!Z-7-7-7-7-79* 

?6#:
:!J


J$9$9E	"**	$M)B)B0?
  .$6"&*
	 F 
r;   c                      [         R                  R                  SSS 5      n U b  U $ [        S[	        5       [        5       S/S5      n[        USS5      $ )Nra   test_assume_fast_crcmodzcrcmod.crcmod_usingExtensionr   F)r!   r   r"   
__import__globalslocalsr   )boto_optnested_crcmods     r9   UsingCrcmodExtensionr9  5  sX    [[__X'@$G(O ih- 
 15	99r;   c                 H   U R                   R                  R                  nU(       d(  UR                  S;   a  gUR	                  S5      (       a  gSUR                  s=::  a  S::  a  O  OUR	                  S5      U l        U R                  n[        U[        5      (       a  UR                  [        5      nUR	                  SS5      nUR	                  S	S5      nUcC  Uc@  U R
                  S
U-  :w  a-  UR                  SU R
                  < SU R                  < 35      egUR                  S:X  ar  UR                  5       nUR                  UR                  UR                  U5      nUR                  S;   a+  [         R"                  R%                  SUR                  -  US9eg)zFReplaces boto.s3.key's should_retry() to handle KMS-encrypted objects.)i  i  Tlocation   i+  etagz/x-amz-server-side-encryption-customer-algorithmNzx-goog-encryption-kms-key-namez"%s"z)ETag from S3 did not match computed MD5. z vs. i  )RequestTimeoutzSaw %s, retrying)responseF)bucket
connectionproviderstatus	getheaderr=  md5
isinstancebytesdecoder   storage_data_errorreadstorage_response_errorreason
error_coder!   	exceptionPleaseRetryException)	r  r?  chunked_transferrB  rE  -amz_server_side_encryption_customer_algorithm goog_customer_managed_encryptionbodyerrs	            r9   _PatchedShouldRetryMethodrU  J  sv   
 [[##,,(	*$*%%HOO"s"""6*DI
((C#uJJtc
 5=4F4F945A1 (0'9'9($(0$5=,4	fsl	"))99dhh01 	1 __
 ==?D

)
)C ~~++NN//
s~~
- 0  
 
r;   c                     [         R                  R                  SSS 5      n [        R                  " [         R
                  R                  R                  R                  5      nU b  U[        R                  " U 5      :H  $ g)Nr=   gs_hostF)	r!   r   r"   r,   r-   gsrA  r	   r   )rW  default_hosts     r9   HasUserSpecifiedGsHostrZ    sY    KKOOM9d;' 2 2 ? ? K KL,3>>'222	r;   c                  8   [         R                  n U R                  " SS5      nU R                  " SS5      =(       a    U R                  " SS5      nU R                  " SS5      =(       a    U R                  " SS5      nU(       + =(       a    U(       + =(       a    U$ )Nr=   r   r>   r   r   r   )r!   r   rC   )r   has_refresh_tokenhas_service_account_credentialshas_hmac_credentialss       r9   UsingGsHmacr_    s     ;;&''7PQ
m%;< @-)>? " m%78 A-)?@   
 (G$G r;   )F)M__doc__
__future__r   r   r   r   r$   r*   r.   rE   r,   r!   r   	boto.authboto.exceptionr   boto.gs.connectionr	   boto.providerr
   boto.pyami.configr   r   r   gslib.exceptionr   gslib.utilsr   gslib.utils.constantsr   r   r   r   gslib.utils.unit_utilr   r   r  oauth2client.clientr   PY3intrs   r"   BOTO_IS_SECURECERTIFICATE_VALIDATION_ENABLEDr#   r3   r:   rG   rK   rM   r\   r_   re   ri   rl   ro   ru   rw   r   r   r}   Httpr   r   r   r   rB   r   r  r!  r%  r   r9  rU  rZ  r_  rJ   r;   r9   <module>rq     sK   ' %  ' 	    
    - + " 1    , # > : 1 & 6 )  *77	$ FK6!'F,I4"Q   'T*86
:$9;PJ &6J #-- /j2B3'T(
upH(E,^:*9vr;   