
    V                       S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKJr  SSKJr  SSKrSS	KJr  SSKrSS
KJr  SSKJ r   SSK!J"r"  SSK#J$r$  SSK%J&r'  SSK(J)r)  SSK*J+r,  SSK-J.r.  SSK/J0r0  SSK1rSSK2rSSK3J4r4  SSK5J6r6  SSK5J7r7  SSK5J8r8  SSK5J9r9  SSK5J:r:  SSK5J;r;  SSK5J<r<  SSK5J=r=  SSK5J>r>  SSK5J?r?  SSK5J@r@  SSK5JArA  SSKBrCSS KDJErE  SS!KDJFrF  SS"KGJHrH  SS#KGJIrI  SS$KJJKrK  SS%KJJLrL  SS&KMJNrO  SS'KPJQrQ  SS(KRJSrS  SS)KRJTrT  SS*KRJUrU  SS+KVJWrW  SS,KVJXrX  SS-KYJZrZ  SS.KYJ[r[  SS/KYJ\r\  SS0KYJ]r]  SS1KYJ^r^  SS2KYJ_r_  SS3K`Jara  SS4K`Jbrb  SS5KcJdrd  SS6KeJfrf  SS7KgJhrh  SS8KiJjrj  SS9KiJkrk  SS:KiJlrl  SS;KiJmrm  SS<KiJnrn  SS=KiJoro  SS>KiJprp  SS?KiJqrq  SS@KiJrrr  SSAKiJsrs  SSBKiJtrt  SSCKiJuru  SSDKiJvrv  SSEKwJxrx  \R                  (       a  \zr{\R                  R                  \R                  R                  \R                  Rz                  \R                  R~                  \R                  R                  \R                  GR                   4r\GR                  " 5       qSFq\GR
                  " SG\GR                  SH9r\GR
                  " SI5      rSJ rSK r " SL SM\5      r " SN SO\95      rg)Pz?XML/boto gsutil Cloud API implementation for GCS and Amazon S3.    )absolute_import)print_function)division)unicode_literalsN)parseString)_exceptions)http_client)config)handler)Cors)LifecycleConfig)CORSConfiguration)DeleteMarker)	Lifecycle)Prefix)Tags)BotoResumableUpload)AccessDeniedException)ArgumentException)BadRequestException)CloudApi)NotEmptyException)NotFoundException)PreconditionException)ResumableDownloadException)ResumableUploadAbortException)ResumableUploadException)!ResumableUploadStartOverException)ServiceException)CommandException)InvalidUrlError)GOOG_PROJ_ID_HDR)PopulateProjectId)GenerationFromUrlAndString)StorageUrlFromString)storage_v1_messages)parallelism_framework_util)ConfigureNoOpAuthIfNeeded)GetMaxRetryDelay)GetNumRetries)ListToGetFields)ValidateDstObjectMetadata)DEFAULT_FILE_BUFFER_SIZE)REQUEST_REASON_ENV_VAR)REQUEST_REASON_HEADER_KEY)S3_DELETE_MARKER_GUID)UTF8)XML_PROGRESS_CALLBACKS)Base64EncodeHash)Base64ToHexHash)AddAcceptEncodingGzipIfNeeded)multiprocessing_context)EncodeStringAsLong)AclTranslation)AddS3MarkerAclToObjectMetadata)CorsTranslation)CreateBucketNotFoundException)%CreateNotFoundExceptionForObjectWrite)CreateObjectNotFoundException)DEFAULT_CONTENT_TYPE)HeadersFromObjectMetadata)LabelTranslation)LifecycleTranslation)REMOVE_CORS_CONFIG)S3MarkerAclFromObjectMetadata)UnaryDictToXml)TWO_MIBFz.*non-\s*existent\s*object)flagsz^"*[a-fA-F0-9]{32}"*$c                     SSSS.nUR                  5        HB  u  p#[        R                  " SUS5      nUc  M"  [        U R                  R
                  X45        MD     g)z:Update Boto Key object with user config's custom endpoint.hostporthost_header)gs_hostgs_portgs_host_headerCredentialsN)itemsr
   getsetattrbucket
connection)keyuser_setting_to_key_attributeuser_settingkey_attributeuser_setting_values        )platform/gsutil/gslib/boto_translation.py_AddCustomEndpointToKeyrZ      s\     %#
 &C%H%H%J!lM<F%cjj##]G &K    c                  \    [         R                  " 5       q[        R                  " SS5      qg)zPerform necessary initialization for multiprocessing.

See gslib.command.InitializeMultiprocessingVariables for an explanation
of why this is necessary.
ir   N)r'   
CreateLockboto_auth_initialized_lockr6   Valueboto_auth_initialized r[   rY   "InitializeMultiprocessingVariablesrc      s&      :DDF177Q?r[   c                   $    \ rS rSrSrS rS rSrg)DownloadProxyCallbackHandler   zFIntermediary callback to keep track of the number of bytes downloaded.c                     Xl         X l        g N)_start_byte	_callback)self
start_bytecallbacks      rY   __init__%DownloadProxyCallbackHandler.__init__   s    !Nr[   c                 f    U R                   (       a   U R                  U R                  U-   U5        gg)zSaves necessary data and then calls the given Cloud API callback.

Args:
  bytes_downloaded: Number of bytes processed so far.
  total_size: Total size of the ongoing operation.
Nrj   ri   )rk   bytes_downloaded
total_sizes      rY   call!DownloadProxyCallbackHandler.call   s*     ~~
nnT%%(88*E r[   rq   N)__name__
__module____qualname____firstlineno____doc__rn   rt   __static_attributes__rb   r[   rY   re   re      s    NFr[   re   c                     ^  \ rS rSrSr       S;U 4S jjrS rS<S jrS=S jr     S>S	 jr	    S?S
 jr
S<S jr     S>S jr   S=S jrS rSSSS\R                   R"                  SSSSSS4S jrS\SS4S jrSS\S4S jr     S>S jr    S?S jr   S=S jr    S?S jrS@S jrS@S jrS r          SAS jr       SBS jr        SCS jr   S=S jr          SDS jr!    S?S jr"S  r#S! r$S" r%S# r&S@S$ jr'S@S% jr(S@S& jr)S@S' jr*S( r+S) r,S* r-S+ r.S, r/S@S- jr0    S?S. jr1    S?S/ jr2SES0 jr3  SFS1 jr4S2 r5S3 r6S4 r7S5 r8S6 r9S7 r:S8 r;S9 r<S:r=U =r>$ )GBotoTranslation   a-  Boto-based XML translation implementation of gsutil Cloud API.

This class takes gsutil Cloud API objects, translates them to XML service
calls, and translates the results back into gsutil Cloud API objects for
use by the caller.

This class does not support encryption and ignores encryption and decryption
parameters. Behavior when encountering encrypted objects is undefined.
TODO: Implement support.

This class does not support handling a Requester Pays user project for
billing, and any given user project will be ignored.
TODO: Support user_project.
Nr   c                 (  > [         [        U ]  UUUUUUUU	S9  Un[           [	        5         [        [        [        5      (       a  SqOS[        l        SSS5        [        R                  R                  SSS5      U l        g! , (       d  f       N5= f)ae  Performs necessary setup for interacting with the cloud storage provider.

Args:
  bucket_storage_uri_class: boto storage_uri class, used by APIs that
                            provide boto translation or mocking.
  logger: logging.logger for outputting log messages.
  status_queue: Queue for relaying status to UI.
  provider: Provider prefix describing cloud storage provider to connect to.
            'gs' and 's3' are supported. Function implementations ignore
            the provider argument and use this one instead.
  credentials: Unused.
  debug: Debug level for the API implementation (0..3).
  http_headers (dict|None): Arbitrary headers to be included in every request. 
  trace_token: Unused in this subclass.
  perf_trace_token: Performance trace token to use when making API calls
      ('gs' provider only).
  user_project: Unused in this subclass
)providerdebughttp_headerstrace_tokenperf_trace_tokenT   NGSUtildefault_api_version1)superr}   rn   r_   r(   
isinstancera   boolvaluebotor
   	get_valueapi_version)rk   bucket_storage_uri_classloggerstatus_queuer   credentialsr   r   r   r   user_project_	__class__s               rY   rn   BotoTranslation.__init__   s    : 
/4)*B*0*63;057C6A;K * M 	A 
$!	)4	0	0 $&'# 
$ {{,,X7L-02D 
$	#s   2B
Bc                     g)z*Service account credentials unused for S3.Nrb   rk   s    rY   GetServiceAccountId#BotoTranslation.GetServiceAccountId   s    r[   c                     UnU R                  U5      nU R                  5       n U R                  UR                  SUS9US9$ ! [         a  nU R                  XqS9   SnAgSnAff = f),See CloudApi class for function doc strings.Tvalidateheadersfieldsbucket_nameN)_StorageUriForBucket_CreateBaseHeaders_BotoBucketToBucket
get_bucketTRANSLATABLE_BOTO_EXCEPTIONS_TranslateExceptionAndRaise)rk   r   r   r   r   
bucket_urir   es           rY   	GetBucketBotoTranslation.GetBucket   s    A**;7J%%'GC%%j&;&;TDK '< 'M-3 & 5 5 ( C
&&q&BCs   A 
A&A!!A&c              #     #    Un[        US9nU R                  5       nU R                  S:X  a  [        U5      U[        '    [
        R                  " SU R                  -  SU R                  U R                  S9nUR                  US9nU HN  n	U R                  S:X  a*  U	R                  R                  5       U	R                  :w  a  M=  U R                  XS9v   MP     g	! [         a  n
U R                  U
5         S	n
A
g	S	n
A
ff = f7f)
r   list_fieldsgsz%s://Fsuppress_consec_slashesr   r   r   s3r   N)r+   r   r   r#   r"   r   storage_urir   r   get_all_bucketsnamelowerr   r   r   )rk   
project_idr   r   r   
get_fieldsr   provider_uribuckets_iterrR   r   s              rY   ListBucketsBotoTranslation.ListBuckets   s     A V4J%%'G}}"3J"?g*%%
DMM
!"'#'#@#@

	l "11'1Bl &==D V[[%6%6%8FKK%G &&v&AA ! ( *
&&q))*s0   >DBC D
D'C=8D=DDc                    UnU R                  U5      n	U R                  5       n
U R                  XZ5         UR                  (       a>  [        R
                  " UR                  5      nU	R                  UR                  5       U
S9  U(       a=  U	R                  5       nX<;  a  [        SU-  5      eU	R                  X9R                  U
S9  U(       a>  U	R                  5       nXL;  a  [        SU-  5      eU	R                  UU	R                  U
S9  UR                  (       aL  UR                  [        :X  a  / Ul        [        R                   " UR                  5      nU	R#                  USU
S9  UR$                  (       a>  [        R
                  " UR$                  5      nU	R'                  UR                  5       U
S9  UR(                  (       a>  [*        R,                  " UR(                  5      n U	R/                  5       R1                  XS9  UR:                  (       a1  [<        R>                  " UR:                  5      nU	RA                  USU
S9  URB                  (       a  URB                  RD                  (       aV  URB                  RF                  (       a;  U	RI                  URB                  RD                  URB                  RF                  SU
S9  OU	RK                  SU
S9  URL                  (       a  U	RO                  URL                  U
S9  URP                  (       a$  U	RS                  URP                  RT                  U
S9  URV                  (       a=  URV                  RX                  nURV                  RZ                  nU	R]                  UUU
S9  U R_                  XS9$ ! [2        R4                  R6                   a  nUR8                  S:w  a  e  SnAGNSnAff = f! [`         a  nU Rc                  XS9   SnAgSnAff = f)r   r   Invalid canned ACL "%s".F   Nr   r   )2r   r   _AddPreconditionsToHeadersaclr8   BotoAclFromMessageset_xml_aclto_xmlcanned_aclsr    set_aclobject_nameset_def_aclcorsrB   r:   BotoCorsFromMessageset_corsdefaultObjectAclset_def_xml_acllabelsr@   BotoTagsFromMessager   set_tagsr   	exceptionGSResponseErrorstatus	lifecyclerA   BotoLifecycleFromMessageconfigure_lifecyclelogging	logBucketlogObjectPrefixenable_loggingdisable_loggingstorageClassset_storage_class
versioningconfigure_versioningenabledwebsitemainPageSuffixnotFoundPageset_website_configr   r   r   )rk   r   metadata
canned_aclcanned_def_aclpreconditionsr   r   r   r   r   boto_aclr   	boto_cors	boto_tagsr   boto_lifecyclemain_page_suffix
error_pages                      rY   PatchBucketBotoTranslation.PatchBucket  sZ    	A**;7J%%'G##M;>C	!44X\\Bx0'B	 ,,.( !;j!HI
I:'='=wO	 ,,., !;n!LM
M~)55'. 	 	0 
==..(-#77F	Iug>		"	"!44X5N5NO""8??#4g"F	$88I		



!
*
*9
*
F 
		-FF &&~ug&N			%%(*:*:*J*J

#
#H$4$4$>$>$,$4$4$D$D$),3 $ 5
 
$
$UG
$
<			$$X%:%:G$L			''(;(;(C(C07 	( 	9			#++::%%22
%%&6&0.5 	& 	7 ^^K^775 ~~-- 	XX_ 	6 ( C
&&q&BCsC   F5O ,N 	FO O8O	O OO 
O:!O55O:c                    UnU R                  U5      nSnU(       a+  UR                  (       a  UR                  R                  5       nU R                  5       n	UR                  S:X  a  [        U5      U	[        '   Sn
U(       a  UR                  (       a  UR                  n
U(       aN  UR                  (       a=  UR                  R                  (       a"  [        UR                  R                  5      U	S'    UR                  U	UU
S9  O UR                  XS9  U R                  XS9$ ! [         a  nU R                  XS9   SnAN-SnAff = f! [         a  nU R                  XS9   SnANSSnAff = f)	r    r   zx-goog-bucket-retention-period)r   locationstorage_classr   N)r   r   r   )r   r   r   r   schemer#   r"   r   retentionPolicyretentionPeriodstrcreate_bucketr   r   r   )rk   r   r   r   r   r   r   r   r   r   r   r   s               rY   CreateBucketBotoTranslation.CreateBucketd  sS    	A**;7JHH%%""((*h %%'GD "3J"?gm	h++ --
x//

"
"
2
247$$445601E  *2/< 	! 	>E   D >>+>55 * E(((DE
 * E(((DEs0   6D' 	E '
E
1EE

E0E++E0c                 x   X24nU R                  U5      nU R                  5       n UR                  US9  g	! [         a  nU R	                  XqS9nU(       a  SUR
                  ;   a   UR                  5       (       a>  U R                  S:X  a  [        SU-  UR                  S9e[        SU-  UR                  S9e[        SU-  UR                  S9e! [         a  n	U R                  XS9   S	n	A	 S	nAg	S	n	A	ff = fU(       a  UR                  S
:X  a  [        SU-  5      eU R                  XqS9   S	nAg	S	nAff = f)r   r   r   BucketNotEmptyr   zVersionedBucketNotEmpty (%s). Currently, gsutil does not support listing or removing S3 DeleteMarkers, so you may need to delete these using another tool to successfully delete this bucket.r   zVersionedBucketNotEmpty (%s)BucketNotEmpty (%s)N  zBucket %s does not exist.)r   r   delete_bucketr   _TranslateBotoExceptionreasonget_versioning_configr   r   r   r   r   )
rk   r   r   r   r   r   r   r   translated_exceptione2s
             rY   DeleteBucketBotoTranslation.DeleteBucket  sY   A**;7J%%'GEw/' E!99
 : &

299
9	H--//}}$%( +66   $.<xx 
 $%3xx  , 	H

*
*2
*
G
G	H$8$?$?3$F ;k IJJ(((D;Es:   6 
D9 &D4'A*C
C8C3*D43C887D44D9c              #     #    Un[        US9nU R                  U5      n	U R                  5       n
USL =(       d    SU;   nUSL =(       d    [        S U 5       5      n U	R	                  U=(       d    SU=(       d    SUU
S9n W GH3  nU(       aR  [        U[        5      (       a=  [        R                  " UR                  [        R                  R                  5      v   M]  U(       d  Mf  Un[        / SQ5      nU(       a  U(       ao  UR                  U5      (       dY  Sn[!        US	S5      (       a  UR"                  n[!        US
S5      (       a  UR$                  nU R'                  UUR                  US9nU R)                  UUS9n[        R                  " U[        R                  R*                  5      v   GM6     g! [
         a  nU R                  XS9   SnAGN[SnAff = f! [
         a  nU R                  XS9   SnAgSnAff = f7f)r   r   Nprefixesc              3   B   #    U  H  oR                  S 5      v   M     g7f)zitems/N)
startswith).0fields     rY   	<genexpr>.BotoTranslation.ListObjects.<locals>.<genexpr>  s      *806u""s   r   )prefix	delimiterall_versionsr   r   )rR   etagr   updated
generationmetagenerationsizer  
version_idr  r   )r+   r   r   anylist_bucketr   r   r   r   r   CsObjectOrPrefixr   CsObjectOrPrefixTypePREFIXsetissubsetgetattrr  r  _GetBotoKey_BotoKeyToObjectOBJECT)rk   r   r  r  r  r   r   r   r   r   r   yield_prefixesyield_objectsobjects_iterr   rT   key_to_convertkey_http_fieldsr  return_objects                       rY   ListObjectsBotoTranslation.ListObjects  s     	A V4J**;7J%%'Gt^;zV';NdN 8c *806*8 '8MC++6<R6?o29E4; , =l(C#jf55))#((*2*G*G*N*NP P].   ! / 
$.$7$7$H$HJsL$//>>jsL$//>>j!--k.1hh9C . EN //7A 0 C- ))-*2*G*G*N*NP PI 	 ( C
&&q&BCT ( C
&&q&BCs\   AH
$F= A$G$ *CG$ <H
=
G!GH
G!!H
$
H.H=H
HH
c                     Un U R                  U R                  UUUS9US9$ ! [         a  nU R                  UUUUS9   SnAgSnAff = f)r   r  r   r   r   r  N)r(  r'  r   r   )rk   r   r   r  r   r   r   r   s           rY   GetObjectMetadata!BotoTranslation.GetObjectMetadata  sv     	A	>""4#3#3K4??I $4 $K +1 # 2 2 ( >
&&q3>3>2< ' >>s   # 
A	AA	c                    ^ U4S j$ )a  Curries a digester object into a form consumable by boto.

Key instantiates its own digesters by calling hash_algs[alg]() [note there
are no arguments to this function].  So in order to pass in our caught-up
digesters during a resumable download, we need to pass the digester
object but don't get to look it up based on the algorithm name.  Here we
use a lambda to make lookup implicit.

Args:
  digester_object: Input object to be returned by the created function.

Returns:
  A function which when called will return the input object.
c                     > T $ rh   rb   )digester_objects   rY   <lambda>0BotoTranslation._CurryDigester.<locals>.<lambda>  s    ?r[   rb   )rk   r8  s    `rY   _CurryDigesterBotoTranslation._CurryDigester  s     #"r[   Fc                    U R                  5       n[        UUS9  U
b  SU	< SU
< 3US'   OU	S:  a	  SU	-  US'   OU	S:  a  SU	-  US'   U(       a_  [        R                  " U5      n[        R                  " [
        R                  " US	   5      5      nU R                  S
:X  a  [        U5        OU R                  XUS9nU(       a3  U R                  S
:X  a#  0 nU H  nU R                  UU   5      UU'   M     O0 nU=(       d    SnU(       a  [        R                  " U5      S   nU(       a!  [        [        U5      [        -  [        5      nO[        n U[        R                   R"                  L a  U R%                  UU	U
UUUUUS9  O@U[        R                   R&                  L a  U R)                  UUUUUUS9  O[+        SU-  5      eU R                  S:X  a  U(       a~   " S S[0        5      nU Hh  nU[        R                   R"                  :X  a  U	S:w  d"  [3        USS5      (       a  UUR4                  ;   d  SUU'   MP  U" UR4                  U   5      UU'   Mj     ggg! [,         a  nU R/                  UUUUS9   SnANSnAff = f)r   )compressed_encodingNzbytes=-ranger   z	bytes=%s-zbytes=%surlr   r  rs   )r   rm   num_callbacks	hash_algs)progress_callbacknum_progress_callbacksr   rC  z Unsupported DownloadStrategy: %sr3  r   c                   $    \ rS rSrSrS rS rSrg)6BotoTranslation.GetObjectMedia.<locals>.HashToDigesterij  zWrapper class to expose hash digests.

boto creates its own digesters in s3's get_file, returning on-the-fly
hashes only by way of key.local_hashes.  To propagate the digest back
to the caller, this stub class implements the digest() function.
c                     Xl         g rh   hash_val)rk   rJ  s     rY   rn   ?BotoTranslation.GetObjectMedia.<locals>.HashToDigester.__init__r  s    $Mr[   c                     U R                   $ rh   rI  r   s    rY   digest=BotoTranslation.GetObjectMedia.<locals>.HashToDigester.digestu  s    == r[   rI  N)rv   rw   rx   ry   rz   rn   rM  r{   rb   r[   rY   HashToDigesterrG  j  s    %!r[   rO  local_hashes)r   r5   jsonloadspicklebinascii
a2b_base64r   rZ   r'  r;  maxintrE   r2   r   DownloadStrategy	RESUMABLE_PerformResumableDownloadONE_SHOT_PerformSimpleDownloadr   r   r   objectr&  rP  )rk   r   r   download_streamr   r  object_sizer>  download_strategyrl   end_byterD  serialization_data	digestersdecryption_tupler   serialization_dictrT   rC  algrs   rE  r   rO  alg_names                            rY   GetObjectMediaBotoTranslation.GetObjectMedia  s   $ %%'G!'6IK*4h?gg	a$
3gg	a#z2gg
 ::&89LL,,-?-FGHc	$	$[*McT]]d*i#,,Ys^<	#  i!J::01,?j"
j/G
#%; =  6>	h77AA	A&&'1'/'*/60A5K1: 	' 	<  9 9 B BB##/#9 	$ 	!   B 1!2 3 	3 }}		!V 	! "H H$=$=$G$GGQnd)K)K)1S5E5E)E #'Ih #11A1A(1K"LIh "! 
  ( >
&&q3>3>2< ' >>s$   3I  	1I  ;I   
J*JJc                 p     UR                  UUUUUS9  g ! [         a    UR                  UUUUS9   g f = f)N)cbnum_cbr   rC  )rk  rl  r   )get_contents_to_file	TypeError)rk   r^  rT   rD  rE  r   rC  s          rY   r\  &BotoTranslation._PerformSimpleDownload  sW    
0	"3&<'.)2	  4
  0	"3&<'.  00s    55c	           
      6   [         R                  [        [        R                  [        R
                  4n	UR                  R                  R                  n
[        5       nSnUn  [        X&5      nUR                  5       nSU;   a  UR                  S5        SX#4-  US'    UR                  UUUR                  USUS9  UR!                  5         g! [         a     UR                  UUUR                  USS9   N:f = f! U	 a  nU
S:  a%  U R"                  R%                  S	['        U5      5        [)        U[        5      (       an  UR*                  [*        R,                  :X  aP  U R.                  S
:X  a"  UR                  UUWR                  USS9   SnAOUR                  UUWR                  USUS9   SnAOSnAf[0        R2                  R4                   a}  nUR6                  [0        R2                  R8                  R:                  :X  a  [5        UR<                  5      eU
S:  a&  U R"                  R%                  SUR<                  5         SnAOSnAff = fUR?                  5       nX-:  a  UnSnOUS-  nX:  a  [5        S5      e URA                  5         O! [         RB                   a     Of = f[E        [F        RF                  " 5       SU-  -  [I        5       5      nU
S:  a  U R"                  R%                  SUU5        [J        RL                  " U5        GM  )a  Downloads bytes from key to fp, resuming as needed.

Args:
  fp: File pointer into which data should be downloaded.
  start_byte: Start byte of the download.
  end_byte: End byte of the download.
  key: Key object from which data is to be downloaded
  headers: Headers to send when retrieving the file
  callback: (optional) a callback function that will be called to report
       progress on the download.  The callback should accept two integer
       parameters.  The first integer represents the number of
       bytes that have been successfully transmitted from the service.  The
       second represents the total number of bytes that need to be
       transmitted.
  num_callbacks: (optional) If a callback is specified with the callback
       parameter, this determines the granularity of the callback
       by defining the maximum number of times the callback will be
       called during the file transfer.
  hash_algs: Dict of hash algorithms to apply to downloaded bytes.

Raises:
  ResumableDownloadException on error.
r   r@  zbytes=%d-%dRange)override_num_retriesrC  )rr  Nr   zCaught exception (%s)r   zBCaught boto.exception.ResumableDownloadException (%s) - will retryzeToo many resumable download attempts failed without progress. You might try this download again later   zWGot retryable failure (%d progress-less in a row).
Sleeping %d seconds before re-trying)'r	   HTTPExceptionIOErrorsocketerrorgaierrorrR   rS   r   r*   re   copypopget_filert   rn  flushr   inforeprr   errnoEPIPEr   r   r   r   dispositionResumableTransferDispositionABORT_CUR_PROCESSmessagetellcloseIncompleteReadminrandomr)   timesleep)rk   fprl   ra  rT   r   rm   rB  rC  retryable_exceptionsr   num_retriesprogress_less_iterationslast_progress_byte
cb_handlerr   sleep_time_secss                    rY   rZ  )BotoTranslation._PerformResumableDownload  s   @ (55w"OO- JJ!!''E/K #
6$1*G
,,.g
++g
(J+AA	/
,,r!$,-!*  , 	
  	/
,,r!$,-	  /	/ " .A:
++

2DG
<a!!agg&<
 ]]d"LL #&./	  1 LL #&./#,  . ^^66 $MMNN77IIJ*1995
5aZKK$$ 779j		('#$  A% 	!	/(@A 	A		''  FMMOq2J/JK,.0o	!'(@	 jj!s sb   %;C= !C ?C= 'C:7C= 9C::C= =IBF6F66!IA3IIJ J/.J/c	                 R   Un	U R                  UUUS9n
U R                  5       n[        X0R                  5      n0 n[	        5       nSn[
        R                  " U5       H#  u  nnSnUc  UR                  U5        M  UUU'   M%     U R                  Xk5        U(       a   U
R                  UUSUS9  UR                  (       a@  [        R                  " UR                  5      n U
R                  UR!                  5       UUS9  U(       a>  U
R#                  5       nUU;  a  [%        SU-  5      eU
R'                  XJR(                  US9  U R+                  UUUUS	9$ ! [         a  nU R                  UUUUS9   SnANSnAff = f! [         a  nU R                  UUUUS9   SnANSnAff = f)
r   r  FTNr   r3  )key_namer   r   )r  r   )_StorageUriForObjectr   r?   r   r$  six	iteritemsaddr   set_metadatar   r   r   r8   r   r   r   r   r    r   r   r4  )rk   r   r   r   r   r  r   r   r   r   
object_urir   meta_headersmetadata_plusmetadata_minusmetadata_changedkvr   r   r   s                        rY   PatchObjectMetadata#BotoTranslation.PatchObjectMetadata  s    	A**;+66@ + BJ %%'G,X}}ELMUNl+1	
1a , 	##M;	@ . %(/ 	  	1 ||228<<@h@x0(3'. 	 	0 **,k	;	&9JFGG%;%;WM!!+"--7)/ " 1 1/ * @((5@5@4> 	) 	@@ * @((5@5@4> 	) 	@@s0   E !F  
E=!E88E= 
F&
F!!F&c           	      ,    UR                  UUUUUSS9  g )NT)md5policyrk  r   rewind)set_contents_from_file)rk   dst_uriupload_streamr  r   rD  r   s          rY   _PerformSimpleUpload$BotoTranslation._PerformSimpleUpload^  s(     ""='**4&7+2*. # 0r[   c                    UR                  5       R                  5       (       a  UR                  UUUUS9  g [        R                  " 5       u  pg [        US5       nUR                  [        5      n	U	(       a/  UR                  U	5        UR                  [        5      n	U	(       a  M/  S S S 5        [        US5       n
UR                  U
UUS9  S S S 5        [        R                  " U5        [        R                  " U5        g ! , (       d  f       Na= f! , (       d  f       NL= f! [        R                  " U5        [        R                  " U5        f = f)N)r  rk  r   wbrb)r  r   )get_providersupports_chunked_transferset_contents_from_streamtempfilemkstempopenreadr-   writer  osr  unlink)rk   r  r  r   rD  r   temp_fh	temp_pathout_fpstream_bytesin_fps              rY   _PerformStreamingUpload'BotoTranslation._PerformStreamingUploadl  s    7799&&}.8*;/6 ' 8 &--/w)T"f&++,DE,LL&(--.FGL  #
 )T"e

(
(0:18 ) : #
 	
		) #"
 #"
 	
		)s=   D$ A
D'D$ ;DD$ 
DD$ 
D!D$ $.Ec	           	      V    [        UU R                  US9n	U	R                  UUUUUUS9  g )N)
resume_url)r   rk  r   )r   r   SendFile)
rk   rT   r  upload_sizetracker_callbackr   rb  rD  r   resumable_uploads
             rY   _PerformResumableUpload'BotoTranslation._PerformResumableUpload  sC     ++;+/;;6HJ c+))3!2&-  /r[   c                    [        U5        U R                  5       nUR                  [        XR                  5      5        UR
                  (       aQ  SU;   a*  US==   SUR
                  R                  S5      -  -  ss'   O!SUR
                  R                  S5      -  US'   UR                  (       aQ  SU;   a*  US==   SUR                  R                  S5      -  -  ss'   O!SUR                  R                  S5      -  US'   SU;   a  US   (       d  SUS'   U R                  X#5        U R                  UR                  UR                  5      nX44$ )	zShared upload implementation.

Args:
  object_metadata: Object metadata describing destination object.
  preconditions: Optional gsutil Cloud API preconditions.

Returns:
  Headers dictionary, StorageUri for upload (based on inputs)
zx-goog-hashz
,crc32c=%s
z	crc32c=%sz,md5=%szmd5=%scontent-typezapplication/octet-stream)r,   r   updater?   r   crc32crstripmd5Hashr   r  rR   r   )rk   object_metadatar   r   r  s        rY   _UploadSetupBotoTranslation._UploadSetup  sK    o.%%'GNN,_mmLM	'	!<#2#9#9#@#@#F$G 	H #."1"8"8"?"?"E#F	'	!9#2#:#:#A#A$#G$H 	I #+"1"9"9"@"@"F#G  )@ :gn##M;''(>(>(7(<(<>Gr[   c           	          U R                  X!5        U R                  5       nU R                  UR                  US9US9$ ! [        R
                  R                   a  nUR                  (       ar  [        R                  UR                  R                  [        5      5      (       a;  [        SR                  [        R                  " SUR                   -  5      5      5      ee SnAff = f)a  Set ACLs on an uploaded object and return its metadata.

Args:
  dst_uri: Generation-specific StorageUri describing the object.
  object_metadata: Metadata for the object, including an ACL if applicable.
  fields: If present, return only these Object metadata fields.

Returns:
  gsutil Cloud API Object metadata.

Raises:
  CommandException if the object was overwritten / deleted concurrently.
r   r   r  zUploaded object (%s) was deleted or overwritten immediately after it was uploaded. This can happen if you attempt to upload to the same object multiple times concurrently.N)_SetObjectAclr   r(  get_keyr   r   InvalidUriErrorr  NON_EXISTENT_OBJECT_REGEXmatchencoder1   r    jointextwrapwrapuri)rk   r  r  r   r   r   s         rY   _HandleSuccessfulUpload'BotoTranslation._HandleSuccessfulUpload  s     2'')g""7??7?#C*0 # 2 2>>)) 		
066qyy7G7G7MNNtyyMMB    	 		s   =A   C'BC""C'c                 *   U R                  5       nUR                  (       a?  [        R                  " UR                  5      nUR	                  UR                  5       US9  gU R                  S:X  a#  [        U5      nU(       a  UR	                  XSS9  ggg)zCSets the ACL (if present in object_metadata) on an uploaded object.r   r   N)r   r   r8   r   r   r   r   rC   )rk   r  r  r   r   s3_acls         rY   r  BotoTranslation._SetObjectAcl  s}    %%'G22?3F3FGh(//+W=	$	,_=f	F4 
 
r[   c                    U(       a  [        S5      eU R                  S:X  a  U R                  UUUUUUS9$ U R                  UUS9u  pU(       d  [	        SU-  5      e U R                  UR                  US9UUUUUUUS9  U R                  UUUS9$ ! [         aP  n[        U R                  UR                  5      nU R                  UUR                  UR                  US	9   S
nAg
S
nAff = f)r   z.XML API does not support gzip-encoded uploads.r   )r   r   r   r  r   z;No tracker callback function set for resumable upload of %sr   )r   rb  rD  r   r   r   r   not_found_exceptionN)NotImplementedErrorr   UploadObjectr  r   r  new_keyr  r   r<   rR   r   r   )rk   r  r  r   r   r  rb  r  rD  encryption_tupler   r   gzip_encodedr   r  r   r  s                    rY   UploadObjectResumable%BotoTranslation.UploadObjectResumable  sB    
:< <}}}.*4-:&,$(  * * ((7D ) FG 79@A B BP
""7??7?#C#0#'#3.86H5F+2 # 4 ))'*917 * 9 9 ( PA
--//1
&&q3B3I3I3B3G3G;N ' PPs   3B 
C-AC((C-c
                 N   U	(       a  [        S5      eU R                  UUS9u  p U R                  UUUUU
S9  U R                  UUUS9$ ! [         aP  n[        U R                  UR                  5      nU R                  UUR                  UR                  US9   SnAgSnAff = f)r   -XML API does not suport gzip-encoded uploads.r  )r   rD  r   r   r  N)
r  r  r  r  r   r<   r   rR   r   r   )rk   r  r  r   rD  r   r  r   r   r  r   r  r   r  s                 rY   UploadObjectStreaming%BotoTranslation.UploadObjectStreaming!  s      OPP((7D ) FGP
""7#0.85F+2	 # 4
 ))'*917 * 9 9 ( PA
--//1
&&q3B3I3I3B3G3G;N ' PPs   #A
 

B$ABB$c           	         U
(       a  [        S5      eU R                  UUS9u  p SnUR                  (       a0  [        UR                  5      UR                  R	                  S5      /nU R                  UUUUUUS9  U R                  UUU	S9$ ! [         aP  n[        U R                  UR                  5      nU R                  UUR                  UR                  US9   SnAgSnAff = f)r   r  r  Nz
"')r  r   rD  r   r   r  )r  r  r  r4   stripr  r  r   r<   r   rR   r   r   )rk   r  r  r   r   r  rD  r  r   r   r  r   r  r  r   r  s                   rY   r  BotoTranslation.UploadObjectB  s     OPP((7D ) FGPc		 	  O334##))'2
  -$'+52C(/   1 ))'*917 * 9 9 ( PA
--//1
&&q3B3I3I3B3G3G;N ' PPs   A'B 
C(AC##C(c                     UnU R                  5       nU R                  X75        U R                  UUUS9n UR                  SUS9  g! [         a  n	U R                  U	UUUS9   Sn	A	gSn	A	ff = f)r   r  Fr   r3  N)r   r   r  
delete_keyr   r   )
rk   r   r   r   r  r   r   r   r  r   s
             rY   DeleteObjectBotoTranslation.DeleteObjectm  s     	A%%'G##M;

#
#K$//9 $ ;C>	nneWn5' >
&&q3>3>2< ' >>s   A 
A-A((A-c           
      t   U
nUb  [        S5      eU R                  UR                  UR                  5      nSnU R                  S:X  a  UnSnU R                  5       nUR                  [        X R                  5      5        U R                  X_5        U(       a  XOUR                  5       R                  '   UR                  (       a  SOSnU R                  S:X  a  [        U5      nU(       a  SnUR                  =(       d    Sn UR                  UR                  UR                  UUUUUS9nU R                  UUS9$ ! [          ap  n[#        U R                  UR                  U R                  UR                  UR                  US9nU R%                  UUR                  UR                  US	9   SnAgSnAff = f)
r   Nz*XML API does not suport max_bytes_per_callr   TF)preserve_aclr   r   src_version_idsrc_generationr   )src_providersrc_bucket_namesrc_object_namer  r  )r  r  rR   r   r   r   r  r?   r   r  
acl_headerr   rC   r   copy_keyr(  r   r<   r   )rk   src_obj_metadatadst_obj_metadatar  r   r   rD  max_bytes_per_callr  rd  r   r   r   r  r  r   r  r  dst_storage_classr  r   r  s                         rY   
CopyObjectBotoTranslation.CopyObject  s    	A% LMM''(8(?(?(8(=(=?G N}}%nn%%'GNN,-=}}MN##M;3=g""$//0+//4UL}},-=>f	 )55=P  !1!8!8!1!6!6.:)0/@0>0> ! @g ""76"::' PA
--

!
!}}*11*//') &&q3C3J3J3C3H3H;N ' PPs   8D= =
F7A&F22F7c                    Un[        U5        UR                  nSUl        UR                  n	SUl        U R                  5       n
U
R	                  [        X R                  5      5        U R                  X:5        UR                  (       d  [        Ul        UR                  U
S'   U R                  X5      n/ nU H9  nU R                  U	UR                  UR                  S9nUR                  U5        M;      UR                  XS9  U R                  U	UUS9$ ! [         a0  nU R!                  XR                  UR                  5         SnAgSnAff = f)r   Nr  r  r   r   )r,   r   rR   r   r  r?   r   r   contentTyper>   r  r  appendcomposer4  r   r   )rk   src_objs_metadatar  r   r  r   r   r   dst_obj_namedst_bucket_namer   r  src_componentssrc_objsrc_urir   s                   rY   ComposeObjectBotoTranslation.ComposeObject  sO    	A./#((L &--O"%%'GNN,-=}}MN##M;''%9" 0 < <gn''FGN$))/*1,,5<5G5G * Ig G$	 %>oono6##O$0+1 $ 3 3 ( >
&&q*A*A'7'<'<> >>s   7D 
E!&EEc                     U(       a]  U R                   S:X  aL  UR                  b  [        UR                  5      US'   UR                  b  [        UR                  5      US'   gggg)z'Adds preconditions (if any) to headers.r   Nzx-goog-if-generation-matchzx-goog-if-metageneration-match)r   	gen_matchr   meta_gen_match)rk   r   r   s      rY   r   *BotoTranslation._AddPreconditionsToHeaders  sc    $.		 	 	,03M4K4K0L,-		%	%	147((5*01 
2 /}r[   c                 d   0 nU R                   (       a  UR                  U R                   5        U R                  S:X  a  U R                  US'   U R                  S:X  a   U R                  (       a  U R                  US'   [
        R                  R                  [        5      nU(       a  X![        '   U$ )z:Creates base headers used for all API calls in this class.r   zx-goog-api-versioncookie)
r   r  r   r   r   r  environrP   r.   r/   )rk   base_headersrequest_reasons      rY   r   "BotoTranslation._CreateBaseHeaders  s    L$++,}}+/+;+;l'(}}!6!6#44l8ZZ^^$:;N0>,-r[   c                     U(       a:  [         R                  U5      (       a  UR                  S5      R                  5       $ gg)zReturns an MD5 from the etag iff the etag is a valid MD5 hash.

Args:
  src_etag: Object etag for which to return the MD5.

Returns:
  MD5 in hex string format, or None.
"'N)	MD5_REGEXsearchr  r   )rk   src_etags     rY   _GetMD5FromETagBotoTranslation._GetMD5FromETag  s6     I$$X..^^E"((** /xr[   c                 |    [         R                  " U R                  < SU< 3SU R                  U R                  SS9$ )zReturns a boto storage_uri for the given bucket name.

Args:
  bucket: Bucket name (string).

Returns:
  Boto storage_uri for the bucket.
://F)r   r   r   r   )r   r   r   r   r   )rk   rR   s     rY   r   $BotoTranslation._StorageUriForBucket  s9     ]]F+ %!%!>!>jj r[   c                     U R                   < SU< SU< 3nU(       a  USU-  -  n[        R                  " USU R                  U R                  S9$ )a  Returns a boto storage_uri for the given object.

Args:
  bucket: Bucket name (string).
  object_name: Object name (string).
  generation: Generation or version_id of object.  If None, live version
              of the object is used.

Returns:
  Boto storage_uri for the object.
r&  /z#%sFr   )r   r   r   r   r   )rk   rR   r   r  
uri_strings        rY   r  $BotoTranslation._StorageUriForObject)  sO     "&DJEJ&&j %!%!>!>jj	 r[   c                     U R                  UUUS9n U R                  5       nUR                  US9nU(       d  [        SU R                  UUUS9eU$ ! [
         a  nU R                  UUUUS9   SnAgSnAff = f)zGets the boto key for an object.

Args:
  bucket_name: Bucket containing the object.
  object_name: Object name.
  generation: Generation or version of the object to retrieve.

Returns:
  Boto key for the object.
r  r   404r3  N)r  r   r  r=   r   r   r   )rk   r   r   r  r  r   rT   r   s           rY   r'  BotoTranslation._GetBotoKey>  s     **;+66@ + BJ>'')gw/c+E,0MM,7,77A	C 	C
 j' >
&&q3>3>2< ' >>s   >A 
A8A33A8c                 <   U R                  UR                  5      n[        R                  " UR                  UR                  S9nU R	                  5       nU R
                  S:X  Ga  U(       a  SU;   a&  [        US5      (       a  UR                  5       Ul        U(       a  SU;   aF   [        R                  " UR                  US95       H  nUR                  R                  U5        M      U(       a  S	U;   a+   UR'                  US9n	[(        R*                  " U	5      Ul        U(       a  S
U;   aF   [        R.                  " UR1                  US95       H  nUR2                  R                  U5        M      U(       a  SU;   aT   UR5                  US9R6                  n
U
(       a3  [        R                  R9                  5       Ul        XR:                  l        U(       a  SU;   a+   UR?                  US9n[@        RB                  " U5      Ul"        U(       a  SU;   a   URG                  US9nU(       a|  SU;   av  US   nSU;   nSU;   nU(       d  U(       aY  [        R                  RI                  5       Ul%        U(       a  US   URJ                  l&        U(       a  US   URJ                  l'        U(       a  SU;   a   URQ                  US9nU(       a|  SU;   av  US   nSU;   nSU;   nU(       d  U(       aY  [        R                  RS                  5       Ul*        U(       a  US   URT                  l+        U(       a  US   URT                  l,        U(       a  SU;   a  UR[                  US9Ul.        U(       a  SU;   a;    UR_                  5       Ra                  5       n[b        Rd                  " U5      Ul3        U(       a  SU;   a  URq                  US9nU(       au  U R
                  S:X  a3  SU;   a-  US   S:X  a$  [        R                  Rs                  SS9Ul:        U$ U R
                  S:X  a"  [        R                  Rs                  SS9Ul:        U$ ! [         a_  nU R                  XqR                  S9nU(       a  [!        U["        5      (       a   SnAGNU R%                  XqR                  S9   SnAGNSnAff = f! [         a$  nU R%                  XqR                  S9   SnAGNSnAff = f! [         a_  nU R                  XqR                  S9nU(       a  [!        U["        5      (       a   SnAGNU R%                  XqR                  S9   SnAGNSnAff = f! [         a$  nU R%                  XqR                  S9   SnAGNSnAff = f! [         a$  nU R%                  XqR                  S9   SnAGNSnAff = f! [         a$  nU R%                  XqR                  S9   SnAGN7SnAff = f! [         a$  nU R%                  XqR                  S9   SnAGNSnAff = f! [h        Rj                  Rl                   a,  nU R
                  S:X  a  URn                  S:X  d  e  SnAGNSnAff = f! [         a$  nU R%                  XqR                  S9   SnAGNSnAff = f)zConstructs an apitools Bucket from a boto bucket.

Args:
  bucket: Boto bucket.
  fields: If present, construct the apitools Bucket with only this set of
          metadata fields.

Returns:
  apitools Bucket.
)r   idr   r   get_storage_classr   r   r   Nr   r   
encryptionr   r   LoggingLogObjectPrefix	LogBucketr   WebsiteConfigurationMainPageSuffixNotFoundPager   r   r   r  r   
VersioningEnabledT)r   );r   r   apitools_messagesBucketr   r   hasattrr1  r   r8   BotoBucketAclToMessageget_aclr   r
  r   r  r   r   r   get_corsr:   BotoCorsToMessager   BotoObjectAclToMessageget_def_aclr   get_encryption_configdefault_kms_key_nameEncryptionValuer2  defaultKmsKeyNameget_lifecycle_configrA   BotoLifecycleToMessager   get_logging_configLoggingValuer   r   r   get_website_configWebsiteValuer   r   r   get_locationr   r   get_tagsr@   BotoTagsToMessager   r   r   StorageResponseErrorr   r  VersioningValuer   )rk   rR   r   r   cloud_api_bucketr   r   r   r	  r   keynamer   boto_logginglogging_configlog_object_prefix_presentlog_bucket_presentboto_websitewebsite_configmain_page_suffix_presentnot_found_page_presentr   r   s                         rY   r   #BotoTranslation._BotoBucketToBucket]  s    **6;;7J(//V[[39;;@%%'G}}~/6.//*0*B*B*D

'u	I#::nnWn-/c  '',/ v'	G ))'):)"1"C"CI"N

 )V3	I#::   13c--44S93 |v-	G55 6 33 !((88: '<C''9 {f,	G%::7:K."99.I 
$ yF*	G#66w6G,i<7))4N(9^(K%!,!>(,>):)A)A)N)N)P&*"#45 !((8#5CK5P ((2 yF*	G#66w6G,4D)*@AN'7>'I$%3~%E"'+A):)A)A)N)N)P&)"#34 !((7'">2 !((5 zV+$.$;$;G$;$L!X'E
	 ++-668)00; 
! \V+33G3Dj	MMT!lj&@|$	1&&66t6D 
%  ]]d"&&66t6D 
% g , 		I!%!=!=[[ "> "*
"-/DEE ,,QKK,H		I , 	G

*
*1++
*
F	G , 		I!%!=!=[[ "> "*
"-/DEE ,,QKK,H		I$ , 	G

*
*1++
*
F	G , 	G

*
*1++
*
F	G  , 	G

*
*1++
*
F	G" , 	G

*
*1++
*
F	G ~~22 	
 --4'AHHO -<	 * E(((DEs   'AQ :*R? 2AS0 AU &*V BV> >BW/  9X  
R<5R7R77R<?
S-	S((S-0
U:5U5UU
V
&VV

V;V66V;>
W,W''W,/
X9XX Y*>!Y%Y- %Y**Y- -
Z7ZZc                    SnU(       a=  SU;   d7  [        U Vs/ s H  oDR                  S5      (       d  M  UPM     sn5      S:  a  U R                  U5      nSnU(       a  SU;   a  [        USS5      nSnU(       a  SU;   a  [        USS5      nSnU(       a  S	U;   a  [        US
S5      nU R	                  U5      nSn	U(       a  SU;   a  U R                  U5      n	Sn
U(       a  SU;   a  U R                  U5      n
SnU(       a  SU;   a%  [        USS5      nU(       a  UR                  S5      nSnU(       a  SU;   aS  [        US5      (       aB  SUR                  ;   a2  [        R                  " UR                  S   5      R                  S5      nSnU(       a  SU;   a  [        US5      (       aC  SUR                  ;   a3  [        R                  " UR                  S   5      R                  S5      nO}U R                  [        USS5      5      (       a%  [        U R                  UR                  5      5      nO7U R                   S:X  a'  U R"                  R%                  SUR                  U5        SnU(       a  SU;   a9  [&        R(                  " [*        R,                  " U[*        R.                  5      5      nSnU(       a  SU;   a  UR0                  =(       d    SnSnU(       a  SU;   a  [        USS5      n[2        R4                  (       aZ  U(       a&  [7        U[8        5      (       a  UR;                  S5      nU(       a&  [7        U[8        5      (       a  UR;                  S5      n[<        R>                  " S$0 SUR@                  RB                  _SURB                  _SU_SURD                  _SURF                  _S URH                  _SU_S	U_SU_SU_SU_S!U_SU	_SU_SU
_SU_SU_SU_6nU RK                  UU5        U(       a  S"U;   a0  [M        [O        U R                   5      U5      nU RQ                  UUUS#9  U$ s  snf )%zConstructs an apitools Object from a boto key.

Args:
  key: Boto key to construct Object from.
  fields: If present, construct the apitools Object with only this set of
          metadata fields.

Returns:
  apitools Object corresponding to key.
Nr   z	metadata/r   cacheControlcache_controlcomponentCountcomponent_countcontentDispositioncontent_dispositionr  timeCreatedr  r  r  cloud_hashes   
r  r  r   zMNon-MD5 etag (%s) present for key %s, data integrity checks are not possible.	mediaLinkr  r   r   _storage_classasciirR   r   contentEncodingcontentLanguager	  r  r   r  rb   ))lenr  _TranslateBotoKeyCustomMetadatar&  _TranslateBotoKeyGeneration_TranslateBotoKeyMetageneration_TranslateBotoKeyTimestampr  r=  rf  base64	b64encoder  r#  r3   r  r   r   warnrT  
b2a_base64rS  dumpsHIGHEST_PROTOCOLr  r  PY3r   bytesdecoder;  ObjectrR   r   content_encodingcontent_languagecontent_type_TranslateDeleteMarkerr$   r%   _TranslateBotoKeyAcl)rk   rT   r   custom_metadatar  r`  rb  rd  r  r  time_createdr  r  md5_hash
media_linkr  r   cloud_api_objectgeneration_strs                      rY   r(   BotoTranslation._BotoKeyToObject  s    OZ6)S"DF5&6&6{&CFD.FIJ.K<<SAoM^v-c?D9mO%/%6=o)V3#C)>E 11#6JN%/;;C@nL]f,44S9lDVv%S&$'d	zz% FX'	n	%	%(c6F6F*F!!#"2"28"<=DDUKHY&(	n	%	%%33C3C*C##C$4$4U$;<CCEJVT :;;#D$8$8$BC==D  	 XXs	, J[F*&&
,,sF33
46jDVv%XX]dM^v- c#3T:m
ww	Jvu--w'	j511??7+(// $zz$XX$ $ ,,	$
 ,,$ $$$ #$ /$ $ $ $ $ &$ '$ !$  !!$" #$$ #%$* 	%56Uf_1
t}}
-z;n
 0+9   ; C 	Es
   QQc                 h   Sn[        USS5      (       a  [        R                  R                  / S9n[        R
                  " UR                  5       H\  u  p4UR                  5       S:X  a  M  UR                  R                  [        R                  R                  R                  UUS95        M^     U$ )zCPopulates an apitools message from custom metadata in the boto key.Nr   )additionalPropertieszcontent-languagerT   r   )r&  r;  r{  MetadataValuer  r  r   r   r  r
  AdditionalProperty)rk   rT   r  r  r  s        rY   rn  /BotoTranslation._TranslateBotoKeyCustomMetadata[  s    OsJ%%)00>>! ? #o---$!779**
,,33$$22EE!LM F O	P	 . r[   c                     SnU R                   S:X  a)  [        USS5      (       a  [        UR                  5      nU$ U R                   S:X  a'  [        USS5      (       a  [	        UR
                  5      nU$ )zFReturns the generation/version_id number from the boto key if present.Nr   r  r   r  )r   r&  longr  r7   r  )rk   rT   r  s      rY   ro  +BotoTranslation._TranslateBotoKeyGenerationj  si    J}}	lD	)	)#..)
  
$		lD	)	)'7
r[   c                 x    SnU R                   S:X  a'  [        USS5      (       a  [        UR                  5      nU$ )z?Returns the metageneration number from the boto key if present.Nr   r  )r   r&  r  r  )rk   rT   r  s      rY   rp  /BotoTranslation._TranslateBotoKeyMetagenerationu  s9    N}}	&	-	-c001r[   c                    UR                   (       ao  SUR                   ;   a  UR                   R                  S5      S-   nOUR                   R                  S5      S-   nSn [        R                  R                  X#5      $ g! [         aA     Sn[        R                  R                  UR                   U5      s $ ! [         a      gf = ff = f)zParses the timestamp from the boto key into an datetime object.

This avoids a dependency on dateutil.

Args:
  key: Boto key to get timestamp from.

Returns:
  datetime object if string is parsed successfully, None otherwise.
.Z000Zz.000000Zz%Y-%m-%dT%H:%M:%S.%fZz%a, %d %b %Y %H:%M:%S %ZN)last_modifiedr  datetimestrptime
ValueError)rk   rT   key_us_timestampfmts       rY   rq  *BotoTranslation._TranslateBotoKeyTimestamp}  s     	!!	!,,33C86A,,33C8:E#c	  ))*:@@   	*#""++C,=,=sC
C 		s0   "B 
C+B;8C;
C	CC		Cc                 X   [        U[        5      (       a  UR                  (       d4  [        R                  R                  5       Ul        / UR                  l        UR                  R                  R                  [        R                  R
                  R                  [        SS95        gg)zCMarks deleted objects with a metadata value (for S3 compatibility).Tr  N)
r   r   r   r;  r{  r  r  r
  r  r0   )rk   rT   r  s      rY   r  &BotoTranslation._TranslateDeleteMarker  s    #|$$&&$5$<$<$J$J$L!9;!!644;;

"
"
0
0
C
C't D 56	 %r[   c                     U R                  UR                  R                  UR                  US9nU R                  5       n U R                  S:X  aG  UR                  US9n[        R                  " U5       H  nUR                  R                  U5        M      U R                  S:X  a  UR                  US9n[        X&5        gg! [        R                  R                   a  nUR                  S:X  a   SnAge SnAff = f)z8Updates cloud_api_object with the ACL from the boto key.r  r   r   r     N)r  rR   r   r   r   r?  r8   rB  r   r
  get_xml_aclr9   r   r   r   r   )	rk   rT   r  r  storage_uri_for_keyr   key_aclr   r   s	            rY   r  $BotoTranslation._TranslateBotoKeyAcl  s    33CJJOO47HH?I 4 K %%'G	$	%--g-> "88AC



%
%c
* B	$	//'/2 	''7A	 

 >>)) 	
S 	s   BC C="C87C88C=c                 <    U R                  UUUUUS9nU(       a  Uee )a  Translates a Boto exception and raises the translated or original value.

Args:
  e: Any Exception.
  bucket_name: Optional bucket name in request that caused the exception.
  object_name: Optional object name in request that caused the exception.
  generation: Optional generation in request that caused the exception.
  not_found_exception: Optional exception to raise in the not-found case.

Raises:
  Translated CloudApi exception, or the original exception if it was not
  translatable.
)r   r   r  r  )r  )rk   r   r   r   r  r  r	  s          rY   r   +BotoTranslation._TranslateExceptionAndRaise  s7    &  77	/ 8 1   r[   c                 f   [        U[        R                  R                  5      (       Ga  UR                  S:X  a)  [        UR                  UR                  UR                  S9$ UR                  S:X  d  UR                  S:X  a)  [        UR                  UR                  UR                  S9$ UR                  S:X  a  U(       a  [        USUR                  5        U$ U(       aI  U(       a!  [        UR                  U R                  UUUS9$ [        UR                  U R                  U5      $ [        UR                  UR                  UR                  S9$ UR                  S:X  aC  UR                  (       a2  S	UR                  ;   a"  [        S
U-  UR                  UR                  S9$ UR                  S:X  a  [!        UR                  5      $ UR                  S:X  a)  [#        UR                  UR                  UR                  S9$ [        U[        R                  R$                  5      (       a  ['        SUR                  UR                  S9$ [        U[        R                  R(                  5      (       a)  ['        UR                  UR                  UR                  S9$ [        U[        R                  R*                  5      (       a^  UR                  (       a8  [,        R/                  UR                  5      (       a  [        UR                  SS9$ [1        UR                  5      $ [        U[        R                  R2                  5      (       a  UR4                  [        R                  R6                  R8                  :X  a  [;        UR                  5      $ UR4                  [        R                  R6                  R<                  :X  a  [!        UR                  5      $ [3        UR                  5      $ [        U[        R                  R>                  5      (       a  [?        UR                  5      $ g)a,  Translates boto exceptions into their gsutil Cloud API equivalents.

Args:
  e: Any exception in TRANSLATABLE_BOTO_EXCEPTIONS.
  bucket_name: Optional bucket name in request that caused the exception.
  object_name: Optional object name in request that caused the exception.
  generation: Optional generation in request that caused the exception.
  not_found_exception: Optional exception to raise in the not-found case.

Returns:
  ServiceException for translatable exceptions, None
  otherwise.

Because we're using isinstance, check for subtypes first.
i  )r   bodyi  r  r  r   r  i  r  r  i  i  zBucket already exists.r  N) r   r   r   rQ  r   r   coder  r   rQ   r=   r   r;   r   r  r   r   r   StorageCreateErrorr   BotoServerErrorr  r  r  r!   r   r  r  ABORTr   
START_OVERr   )rk   r   r   r   r  r  s         rY   r  'BotoTranslation._TranslateBotoException  s   * !T^^8899	
S"166!((HH88s?ahh#o$QVVAHH166JJ88s? %x
:$
$0151<1<<F	H H
 /qxx/:< < 188!&&II88s?qvv*:aff*D !6!D()&'ff. 	. 88s? 1;;88s?$QVVAHH166JJ!T^^66776%&XX#$66+ + !T^^3344aiiqvvFF!T^^3344	
066qyyAA 377QYY''!T^^<<==	
$..EEKK	K,QYY77MMNN77BBC0;;'		22!T^^>>??'		22r[   c                     [         R                  " UR                  SU R                  U R                  S9nU(       a  UR                  5       $ UR                  5       $ ! [         a  nU R                  U5         SnAgSnAff = f5See CloudApiDelegator class for function doc strings.Fr   N)	r   r   
url_stringr   r   rC  r?  r   r   )rk   storage_urldef_obj_aclr  r   s        rY   XmlPassThroughGetAcl$BotoTranslation.XmlPassThroughGetAcl/  sr    *

 
 "'#'#@#@

	c
 
  {{}' *
&&q))*s   AA A 
B(A>>Bc                 l   U R                  5       n [        R                  " UR                  SU R                  U R
                  S9nU(       a  U(       a>  UR                  5       nX;  a  [        SU-  5      eUR                  XR                  US9  gUR                  5       nX;  a  [        SU-  5      eUR                  XR                  US9  gU(       a  UR                  XR                  US9  gUR                  XR                  US9  g! [         a  nU R                  U5         SnAgSnAff = f)r  Fr   r   r   N)r   r   r   r  r   r   r   r    r   r   r   r   r   r   r   )	rk   acl_textr  cannedr  r   r  r   r   s	            rY   XmlPassThroughSetAcl$BotoTranslation.XmlPassThroughSetAcl>  s    %%'G*

 
 "'#'#@#@

	c
 
)+("#=#HII
//(OOW/
E)+("#=#HII
++h+
A


h

I
//(OOW/
E' *
&&q))*s*   B D =D !D 3D 
D3D..D3c           
      t   U R                  5       nUR                  S:X  a  [        5       nO
[        5       n[        R
                  " US5      n [        R                  R                  X5         [        R                   " UR"                  SU R$                  U R&                  S9nUR)                  USUS9  g! [        R                   aF  n[        SUR                  5       < SUR                  5       < SUR                  5       < 35      eSnAff = f! [*         a  nU R-                  U5         SnAgSnAff = f)	r  r   NzRequested CORS is invalid: 	 at line 	, column Fr   r   )r   r   S3Corsr   r   
XmlHandlerxmlsaxr   SaxExceptionsSAXParseExceptionr    
getMessagegetLineNumbergetColumnNumberr   r   r  r   r   r   r   r   )rk   	cors_textr  r   cors_objhr   r  s           rY   XmlPassThroughSetCors%BotoTranslation.XmlPassThroughSetCors_  s   %%'GT!hh8T*AD	gg)'*

 
 "'#'#@#@

	c
 
ll8UGl4 ** D <<>1??,a.?.?.ACD DD ( *
&&q))*2   B5 .AD 5D	AD

D
D7D22D7c                    U R                  5       n[        R                  " UR                  SU R                  U R
                  S9n UR                  SUS9n[        R                  R                  R                  WR                  5       R                  [        5      5      nUR!                  SS9$ ! [         a  nU R                  U5         SnANySnAff = fr  Fr   r   Nz    )indent)r   r   r   r  r   r   r@  r   r   r  domminidomr   r   r  r1   toprettyxml)rk   r  r   r  r   r   
parsed_xmls          rY   XmlPassThroughGetCors%BotoTranslation.XmlPassThroughGetCorsz  s    %%'G


 %!%!>!>jj	C
*\\%\1d ,,T[[]-A-A$-GHJ!!!00 ( *
&&q))*s   B0 0
C:CCc                    U R                  5       n [        R                  " UR                  SU R                  U R
                  S9nUR                  SUS9n[        R                  R                  R                  WR                  5       R                  [        5      5      nUR!                  SS9$ ! [         a  nU R                  U5         SnANySnAff = fr  )r   r   r   r  r   r   rH  r   r   r  r  r  r   r   r  r1   r  )rk   r  r   r  r   r   r  s          rY   XmlPassThroughGetLifecycle*BotoTranslation.XmlPassThroughGetLifecycle  s    %%'G*

 
 "'#'#@#@

	c
 **5'*Bi ,,Y-=-=-?-F-Ft-LMJ!!!00 ( *
&&q))*s   AB0 0
C:CCc           
      t   U R                  5       nUR                  S:X  a  [        5       nO
[        5       n[        R
                  " US5      n [        R                  R                  X5         [        R                   " UR"                  SU R$                  U R&                  S9nUR)                  USUS9  g! [        R                   aF  n[        SUR                  5       < SUR                  5       < SUR                  5       < 35      eSnAff = f! [*         a  nU R-                  U5         SnAgSnAff = f)	r  r   Nz'Requested lifecycle config is invalid: r  r  Fr   r   )r   r   S3Lifecycler   r   r  r  r  r   r  r  r    r  r  r  r   r   r  r   r   r   r   r   )rk   lifecycle_textr  r   lifecycle_objr  r   r  s           rY   XmlPassThroughSetLifecycle*BotoTranslation.XmlPassThroughSetLifecycle  s   %%'GT!!mm%'m=$/AD	gg.,*

 
 "'#'#@#@

	c
 
mUGD ** D<<>1??,a.?.?.ACD DD ( *
&&q))*r  c                 @   U R                  5       n [        R                  " UR                  SU R                  U R
                  S9n[        UR                  US95      n[        W5      R                  5       $ ! [         a  nU R                  U5         SnAN9SnAff = fr  Fr   r   N)r   r   r   r  r   r   rD   rJ  r   r   XmlParseStringr  )rk   r  r   r  logging_config_xmlr   s         rY   XmlPassThroughGetLogging(BotoTranslation.XmlPassThroughGetLogging  s    %%'G	*

 
 "'#'#@#@

	c
 *

 
 
 
13
 ,-99;; ( *
&&q))*   AA8 8
BBBc                 L   U R                  5       n [        R                  " UR                  SU R                  U R
                  S9nUR                  5       R                  5       n[        W5      R                  5       $ ! [         a  nU R                  U5         SnAN9SnAff = fr  )r   r   r   r  r   r   r   get_xml_tagsr   r   r  r  )rk   r  r   r  tagging_config_xmlr   s         rY   XmlPassThroughGetTagging(BotoTranslation.XmlPassThroughGetTagging  s    %%'G
*

 
 "'#'#@#@

	c >>+88: ,-99;; ( *
&&q))*s   AA> >
B#BB#c           
      V   U R                  5       n[        5       n[        R                  " US5      n [        R
                  R                  X5         [        R                  " UR                  SU R                   U R"                  S9nUR%                  5       R'                  XCS9  g! [        R                   aF  n[        SUR                  5       < SUR                  5       < SUR                  5       < 35      eSnAff = f! [(         a  nU R+                  U5         SnAgSnAff = f)r  Nz,Requested labels/tagging config is invalid: r  r  Fr   r   )r   r   r   r  r  r  r   r  r  r    r  r  r  r   r   r  r   r   r   r   r   r   )rk   tagging_textr  r   tags_objr  r   r  s           rY   XmlPassThroughSetTagging(BotoTranslation.XmlPassThroughSetTagging  s    %%'GvH8T*AK	gg,*
*

 
 "'#'#@#@

	c 
nn: ** K,,.!//"3Q5F5F5HJK KK ( *
&&q))*s1   B& AD &D :AC;;D 
D(D##D(c                 @   U R                  5       n [        R                  " UR                  SU R                  U R
                  S9n[        UR                  US95      n[        W5      R                  5       $ ! [         a  nU R                  U5         SnAN9SnAff = fr  )r   r   r   r  r   r   rD   rL  r   r   r  r  )rk   r  r   r  web_config_xmlr   s         rY   XmlPassThroughGetWebsite(BotoTranslation.XmlPassThroughGetWebsite  s    %%'G*

 
 "'#'#@#@

	c
 &c&<&<W&<&MNn .)5577 ( *
&&q))*r  )r   )NNr   NNNN)NN)NNN)NNNNN)NNNNrh   )
NNNNNNNNNF)NNNNNNF)NNNNNNNF)	NNNNNNNNN)F)TF)?rv   rw   rx   ry   rz   rn   r   r   r   r   r   r  r0  r4  r;  r   rX  r[  rh  r2   r\  rZ  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r#  r   r  r'  r   r(  rn  ro  rp  rq  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r{   __classcell__)r   s   @rY   r}   r}      s   &   $ /2b
C*6 "!% $KC^ #  #6J$EP  #?CH $(!%#>&#* # $!%).'/'@'@'I'I !"'+(,#&*qMl 044J%)'+00 )-)-.D*.B"P &*%)(,#'!%>1F  $&*-1#'0" *.04&*D *.1504&*/&%NB	5 (,*.!%/3-1.2-1%)#').0Pj (,.2*.-1%)#').PH #!%%)$(  %)P\ "&" >2 !% ##'$("&"&DPR #'%)!)>V* 
+ *>>L\n`	86: /3.2-16:@ +/*.)-26Pf*$ #'',	*B*61"1"*4< <"*08 8r[   r}   )rz   
__future__r   r   r   r   rr  rT  r  r  rQ  r  rS  r  rerv  r  r  	threadingr  r  xml.dom.minidomr   r  xml.saxr   r  r  	six.movesr	   r   r
   r   boto.gs.corsr   boto.gs.lifecycler   boto.s3.corsr   r  boto.s3.deletemarkerr   boto.s3.lifecycler   r  boto.s3.prefixr   boto.s3.taggingr   boto.exception
boto.utilsgslib.boto_resumable_uploadr   gslib.cloud_apir   r   r   r   r   r   r   r   r   r   r   r   gslib.devshell_auth_plugingslibgslib.exceptionr    r!   gslib.project_idr"   r#   gslib.storage_urlr$   r%   "gslib.third_party.storage_apitoolsr&   r;  gslib.utilsr'   gslib.utils.boto_utilr(   r)   r*   gslib.utils.cloud_api_helperr+   r,   gslib.utils.constantsr-   r.   r/   r0   r1   r2   gslib.utils.hashing_helperr3   r4   gslib.utils.metadata_utilr5   &gslib.utils.parallelism_framework_utilr6   gslib.utils.text_utilr7   gslib.utils.translation_helperr8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   gslib.utils.unit_utilrE   rx  rW  r  r   r  r  r  rQ  r   Lockr_   ra   compileDOTALLr  r   rZ   rc   r]  re   r}   rb   r[   rY   <module>r     s   F & %  '      	   	      
 9 0 
 !     - 4 - 6 !     ; 1 - / $ - - 1 6 9 4 = , ! , + - . 8 2 W 2 ; 2 / 8 B : 8 ; 7 & 8 7 6 C J 4 9 I : H P H ? D ; ? = H 9 )77	$ $ > > $ > > $ I I $ G G $ A A $ C C E  '^^-  JJ'D-/YY8  JJ/0	
H	@F6 F$N8h N8r[   