
    f                        S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	J
r
  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr   SSK!J"r"  SSK#J$r$  SSK%J&r&  SSK%J'r'  SSK(J)r)  SSK(J*r*  SSK(J+r+  SSK(J,r,  SSK-J.r.  SSK-J/r/  SSK0J1r1  SSK2J3r3  SSK2J4r4  SS K5J6r6  SS!K7J8r8  S"r9S#r:S$\:-   S%-   r;S& r<S.S' jr=S( r> " S) S*\?5      r@ " S+ S,\5      rAS- rBg)/zIImplementation of rewrite command (in-place cloud object transformation).    )absolute_import)print_function)division)unicode_literalsN)encoding)config)EncryptionException)Command)CommandArgument)ApiSelector)CommandException)NameExpansionIterator)SeekAheadNameExpansionIterator)FileProgressCallbackHandler)StorageUrlFromString)storage_v1_messages)FileMessage)GetCloudApiInstance)NO_MAX)UTF8)CryptoKeyType)CryptoKeyWrapperFromKey)GetEncryptionKeyWrapper)MAX_DECRYPTION_KEYS)GcloudStorageFlag)GcloudStorageMap)StdinIterator)ConvertRecursiveToFlatWildcard)NormalizeStorageClass)	text_util)PreconditionsFromHeadersA   z
  gsutil rewrite -k [-O] [-f] [-r] [-s] url...
  gsutil rewrite -k [-O] [-f] [-r] [-s] -I
  gsutil rewrite -s [-k] [-O] [-f] [-r] url...
  gsutil rewrite -s [-k] [-O] [-f] [-r] -I
z
<B>SYNOPSIS</B>
a  


<B>DESCRIPTION</B>
  The gsutil rewrite command rewrites cloud objects, applying the specified
  transformations to them. The transformation(s) are atomic for each affected
  object and applied based on the input transformation flags. Object metadata
  values are preserved unless altered by a transformation. At least one
  transformation flag, -k or -s, must be included in the command.

  The -k flag is supported to add, rotate, or remove encryption keys on
  objects.  For example, the command:

    gsutil rewrite -k -r gs://bucket

  updates all objects in gs://bucket with the current encryption key
  from your boto config file, which may either be a base64-encoded CSEK or the
  fully-qualified name of a Cloud KMS key.

  The rewrite command acts only on live object versions, so specifying a
  URL with a generation number fails. If you want to rewrite a noncurrent
  version, first copy it to the live version, then rewrite it, for example:

    gsutil cp gs://bucket/object#123 gs://bucket/object
    gsutil rewrite -k gs://bucket/object

  You can use the -s option to specify a new storage class for objects.  For
  example, the command:

    gsutil rewrite -s nearline gs://bucket/foo

  rewrites the object, changing its storage class to nearline.

  If you specify the -k option and you have an encryption key set in your boto
  configuration file, the rewrite command skips objects that are already
  encrypted with the specified key.  For example, if you run:

    gsutil rewrite -k -r gs://bucket

  and gs://bucket contains objects encrypted with the key specified in your boto
  configuration file, gsutil skips rewriting those objects and only rewrites
  objects that are not encrypted with the specified key. This avoids the cost of
  performing redundant rewrite operations.

  If you specify the -k option and you do not have an encryption key set in your
  boto configuration file, gsutil always rewrites each object, without
  explicitly specifying an encryption key. This results in rewritten objects
  being encrypted with either the bucket's default KMS key (if one is set) or
  Google-managed encryption (no CSEK or CMEK). Gsutil does not attempt to
  determine whether the operation is redundant (and thus skippable) because
  gsutil cannot be sure how the object is encrypted after the rewrite. Note that
  if your goal is to encrypt objects with a bucket's default KMS key, you can
  avoid redundant rewrite costs by specifying the bucket's default KMS key in
  your boto configuration file; this allows gsutil to perform an accurate
  comparison of the objects' current and desired encryption configurations and
  skip rewrites for objects already encrypted with that key.

  If have an encryption key set in your boto configuration file and specify
  multiple transformations, gsutil only skips those that would not change
  the object's state. For example, if you run:

    gsutil rewrite -s nearline -k -r gs://bucket

  and gs://bucket contains objects that already match the encryption
  configuration but have a storage class of standard, the only transformation
  applied to those objects would be the change in storage class.

  You can pass a list of URLs (one per line) to rewrite on stdin instead of as
  command line arguments by using the -I option. This allows you to use gsutil
  in a pipeline to rewrite objects identified by a program, such as:

    some_program | gsutil -m rewrite -k -I

  The contents of stdin can name cloud URLs and wildcards of cloud URLs.

  The rewrite command requires OWNER permissions on each object to preserve
  object ACLs. You can bypass this by using the -O flag, which causes
  gsutil not to read the object's ACL and instead apply the default object ACL
  to the rewritten object:

    gsutil rewrite -k -O -r gs://bucket


<B>OPTIONS</B>
  -f            Continues silently (without printing error messages) despite
                errors when rewriting multiple objects. If some of the objects
                could not be rewritten, gsutil's exit status is non-zero even
                if this flag is set. This option is implicitly set when running
                "gsutil -m rewrite ...".

  -I            Causes gsutil to read the list of objects to rewrite from stdin.
                This allows you to run a program that generates the list of
                objects to rewrite.

  -k            Rewrite objects with the current encryption key specified in
                your boto configuration file. The value for encryption_key may
                be either a base64-encoded CSEK or a fully-qualified KMS key
                name. If no value is specified for encryption_key, gsutil
                ignores this flag. Instead, rewritten objects are encrypted with
                the bucket's default KMS key, if one is set, or Google-managed
                encryption, if no default KMS key is set.

  -O            When a bucket has uniform bucket-level access (UBLA) enabled,
                the -O flag is required and skips all ACL checks. When a
                bucket has UBLA disabled, the -O flag rewrites objects with the
                bucket's default object ACL instead of the existing object ACL.
                This is needed if you do not have OWNER permission on the
                object.

  -R, -r        The -R and -r options are synonymous. Causes bucket or bucket
                subdirectory contents to be rewritten recursively.

  -s <class>    Rewrite objects using the specified storage class.
c                     U R                   (       d$  U R                  R                  [        U5      5        U =R                  S-  sl        g)z9Simple exception handler to allow post-completion status.   N)continue_on_errorloggererrorstrop_failure_count)clses     )platform/gsutil/gslib/commands/rewrite.py_RewriteExceptionHandlerr-      s2    			JJSV!    c                 "    U R                  XS9  g )Nthread_state)RewriteFunc)r*   name_expansion_resultr1   s      r,   _RewriteFuncWrapperr4      s    //'/Cr.   c              #   n   #    U  H+  n[        U5      R                  b  [        SU-  5      eUv   M-     g7f)zAGenerator function that ensures generation-less (live) arguments.Nz-"rewrite" called on URL with generation (%s).)r   
generationr   )url_strsurl_strs     r,   GenerationCheckGeneratorr9      s=     gG$//;L$% & &
M	 s   35c                        \ rS rSrSrSrSrSrg)_TransformTypes   z Enum class for valid transforms.
crypto_keystorage_class N)__name__
__module____qualname____firstlineno____doc__
CRYPTO_KEYSTORAGE_CLASS__static_attributes__r?   r.   r,   r;   r;      s    (*!-r.   r;   c                      \ rS rSrSr\R                  " S/ \S\SSSS\	R                  /\	R                  \R                  " 5       /S9r\R                  " SSS	/S
S\0 S9r\" / SQ\" S5      \" S5      \R(                  " SSS5      (       a  SO\" S5      \" S5      \" S5      \" S5      \" S5      S.S9rS rS rSS jrSrg)RewriteCommand   z)Implementation of gsutil rewrite command.rewriter   zfkIrROs:F)command_name_aliasesusage_synopsismin_argsmax_argssupported_sub_argsfile_url_okprovider_url_okurls_start_arggs_api_supportgs_default_apiargparse_argumentsrekeyrotatecommand_helpzRewrite objects)	help_namehelp_name_aliases	help_typehelp_one_line_summary	help_textsubcommand_help_text)storageobjectsupdate-Iz--continue-on-errorGSUtilencryption_keyNz--clear-encryption-keyz--no-preserve-acl-r-s)rc   -f-k-Orf   -Rrg   )gcloud_commandflag_mapc                 R    UR                   S:w  a  [        S[        U5      -  5      eg )Ngsz5"rewrite" called on URL with unsupported provider: %s)schemer   r(   )selfurls     r,   CheckProviderRewriteCommand.CheckProvider  s.    
zzT
ACH
LN N r.   c                 	   U R                   U l        0 U l        SU l        SU l        SU l        SS/U l        [        5       U l        SU l	        [        [        5      U l        U R                  (       a  U R                  R                  OSU l        U R                  (       a  U R                   H  u  pUS:X  a	  SU l        M  US:X  a+  U R                  R!                  ["        R$                  5        ME  US:X  a	  SU l        MT  US	:X  a	  SU l        Mc  US
:X  d  US:X  a  SU l        SU l        M  US:X  d  M  U R                  R!                  ["        R*                  5        [-        U5      U l        M     U R
                  (       a'  U R.                  (       a  [1        S5      e[3        5       nO(U R.                  (       d  [1        S5      eU R.                  nU R                  (       d  [1        SU R                  -  5      e[5        U R6                  =(       d    0 5      U l        [;        U5      nU R&                  (       a  [=        U5      n[?        U R@                  U RB                  U RD                  U RF                  UU R&                  U RH                  U R                  =(       d    U R                   SS/S9	nSnU R
                  (       dZ  [=        U5      n[K        U R@                  U RB                  U RM                  5       UU R&                  U R(                  U RH                  S9n[O        S[P        5       Hr  nUS-   n	[S        [        RT                  " SS[W        U	5      -  S5      5      n
U
c    O<U
RX                  [Z        R\                  :X  d  MZ  XR                  U
R                  '   Mt     U R                  b#  U R                  U R                  U R                  '   U R                  c@  SR_                  [`        Rb                  " S5      5      n[e        SU-  [f        Rh                  S9  U Rk                  [l        U[n        U R                  (       + S/US9  U R                  (       a/  U R                  (       a  SOSn[1        SU R                  X4-  5      eg)z,Command entry point for the rewrite command.NFri   rg   r   rh   Trc   rj   rf   rk   z&No arguments allowed with the -I flag.z:The rewrite command (without -I) expects at least one URL.zgrewrite command requires at least one transformation flag. Currently supported transformation flags: %snamesize)
project_idr%   bucket_listing_fields)all_versionsrx   r$   rd   zdecryption_key%s
a>  NOTE: No encryption_key was specified in the boto configuration file, so gsutil will not provide an encryption key in its rewrite API requests. This will decrypt the objects unless they are in buckets with a default KMS key set, in which case the service will automatically encrypt the rewritten objects with that key.z%s
)filer)   )fail_on_errorshared_attrsseek_ahead_iterators z*%d file%s/object%s could not be rewritten.)8parallel_operationsr%   csek_hash_to_keywrapperdest_storage_classno_preserve_aclread_args_from_stdinsupported_transformation_flagssettransform_typesr)   r   r   boto_file_encryption_keywrappercrypto_key_sha256boto_file_encryption_sha256sub_optsaddr;   rE   recursion_requestedrz   rF   r   argsr   r   r!   headerspreconditionsr9   r   r   command_namedebugr&   
gsutil_apirx   r   GetSeekAheadGsutilApiranger   r   getr(   crypto_typer   CSEKjointextwrapwrapprintsysstderrApplyr4   r-   )rq   oar7   url_strs_generatorname_expansion_iteratorr   seek_ahead_url_strsi
key_number
keywrappermsg
plural_strs                r,   
RunCommandRewriteCommand.RunCommand  s   !55D#%D "D D %D+/,D'5DD+B6+JD( // 	,,>>59 	$ }}--$!9#'$
 $Y



"
"?#=#=
>$Y&*$
#$Y!%$
$Y!t)%)$
""$
$Y



"
"?#@#@
A$9!$<$
!      	GHHhYY  0 1 	1h9

-
-./ /
 2$,,2D"ED1(; 9:LM 4

  ??00LD4L4L%v.	0  $$ ;8D:



**

$
$
&


"
"((__& 1)*q5j*
**X1C
OCT
JLj					=#5#5	5EO$$Z%A%AB + ''3

.
. ""4#C#CD ++3II
--PQc FSLszz* 	JJ"&'"&"8"88/0#6  8 //3RjI"33ZLM N N r.   c                 
   [        XS9nUR                  nU R                  U5        UR                  UR                  UR
                  UR                  UR                  S9nU R                  (       a  / Ul	        OUR                  (       d  [        SU-  5      eUR                  (       a  UR                  OS nS nUR                  (       aB  UR                  R                  (       a'  UR                  R                  nUR                  S5      nUS L=(       d    US LnS n	U R                  b>  U R                  R                   ["        R$                  :X  a  U R                  R&                  n	S n
U R                  b>  U R                  R                   ["        R(                  :X  a  U R                  R*                  n
U R                  S LnXz:H  =(       a    Xi:H  n[,        R.                  U R0                  ;  a  U(       d  [3        SU< SU< S35      e/ n[,        R4                  U R0                  ;   a4  U R6                  [9        UR:                  5      :X  a  UR=                  S5        [,        R.                  U R0                  ;   a  U(       a  U(       a  UR=                  S	5        [?        U5      [?        U R0                  5      :X  a$  U R@                  RC                  S
U< SU< 35        g [D        RF                  " [H        RJ                  [D        RL                  " U5      5      nS Ul        S Ul'        S Ul        S Ul        [,        R4                  U R0                  ;   a  U R6                  Ul        U	b  Xl        S nUb2  XpRP                  ;   a  U RP                  U   nO[3        SU< SU< 35      eSn[,        R.                  U R0                  ;   a9  U(       a  U(       a
  U(       d  SnO!U(       a
  U(       d  SnOU(       d	  U(       a  Sn[R        RT                  RW                  [Y        UURZ                  5      5        [R        RT                  R]                  5         UR^                  Ra                  [c        US [d        Rd                  " 5       SURf                  [b        Rh                  S95        [k        UR^                  UUS9Rl                  nURo                  UUUR                  U Rp                  UUU R                  UR                  / S9	  UR^                  Ra                  [c        US [d        Rd                  " 5       SURf                  [b        Rh                  S95        g )Nr0   )r6   providera  No OWNER permission found for object %s. If your bucket has uniform bucket-level access (UBLA) enabled, include the -O option in your command to avoid this error. If your bucket does not use UBLA, you can use the -O option to apply the bucket's default object ACL when rewriting.asciizThe "-k" flag was not passed to the rewrite command, but the encryption_key value in your boto config file did not match the key used to encrypt the object "z	" (hash: zO). To encrypt the object using a different key, you must specify the "-k" flag.zstorage classzencryption keyz	Skipping z&, all transformations were redundant: z(Missing decryption key with SHA256 hash z#. No decryption key matches object 	RewritingRotating
Decrypting
EncryptingF)finishedrw   message_type)src_urloperation_name)src_generationr   progress_callbackdecryption_tupleencryption_tupler   fieldsT)9r   expanded_storage_urlrs   GetObjectMetadatabucket_nameobject_namer6   rp   r   aclr   
kmsKeyNamecustomerEncryption	keySha256encoder   r   r   CMEKr=   r   r   r;   rE   r   r	   rF   r   r   storageClassappendlenr&   infor   PyValueToMessageapitools_messagesObjectMessageToPyValueidr   r   r   write_ConstructAnnounceText
url_stringflushstatus_queueputr   timerw   FILE_REWRITEr   call
CopyObjectr   )rq   r3   r1   r   transform_urlsrc_metadatasrc_encryption_kms_keysrc_encryption_sha256src_was_encrypteddest_encryption_kms_keydest_encryption_sha256should_encrypt_destencryption_unchangedredundant_transformsdest_metadatadecryption_keywrapperr   r   s                     r,   r2   RewriteCommand.RewriteFunc  s   $TEJ)>>M}% //!!!! ++%%	 0 'L l ,	,- - ".!8!8 +55>B  !''''11*==GG3::7C.d: </t;  #,,8,,88M<N<NN $ D D O O!,,8,,88M<N<NN

.
.
@
@  >>dJ1K N2M  	""$*>*>> 
 /	12 2  	%%)=)==#8%%$' 	'!!/2 	""d&:&:: 4!!"23
 C(<(<$==
kk%';= > --  (";";L"IKM  $M M (,M$#M $$(<(<<#'#:#:m *!8 !(	">">	>(()>? 	 "#8-IJ 	J !N!!T%9%99	2#%. %8% %8% JJ~}/G/GHJJJ MIIK"%**!,!9!9	;< 4%' (,t 
 ,')6)A)A(,(:(:,=+@+/+O+O#0#7#7!#  % MIIK!%**!,!9!9	;<r.   )rz   r   r   r%   r   r   r   r)   r   r   r   r   r   N)r@   rA   rB   rC   rD   r
   CreateCommandSpec	_SYNOPSISr   r   JSONr   MakeZeroOrMoreCloudURLsArgumentcommand_specHelpSpec_DETAILED_HELP_TEXT	help_specr   r   r   r   gcloud_storage_maprs   r   r2   rG   r?   r.   r,   rI   rI      s    1 **#!&&' %%)IIKLN,  (+-#) (5  % 56 jj+;TBBd 89 34%%%!,N
B{<r.   rI   c                    U SS R                  S5      n[        U5      n[        S5      nU[        U5      -   U-   [        :  a!  [        S5      nSU[        U-
  U-
  U-
  * S -  nU< U< S3nS[        -  nUR                  U5      $ )	a  Constructs announce text for ongoing operations on url_string.

This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS, and
informs the rewrite-related operation ('Encrypting', 'Rotating', or
'Decrypting').

Args:
  operation_name: String describing the operation, i.e.
      'Rotating' or 'Encrypting'.
  url_string: String describing the file/object being processed.

Returns:
  Formatted announce text for outputting operation progress.
N
      z: z...z...%s:z{0:%ds})ljustr   MAX_PROGRESS_INDICATOR_COLUMNSformat)r   r   justified_op_string	start_lenend_lenellipsis_lenbase_announce_text
format_strs           r,   r   r   E  s    & 's+11"5%&)I'#j/!G+.LLu:L:(F(1)24;)<>J)K 'L 'M N NJ"5zB99*			-	..r.   r   )CrD   
__future__r   r   r   r   r   r   r   apitools.base.pyr   botor   gslib.cloud_apir	   gslib.commandr
   gslib.command_argumentr   gslib.cs_api_mapr   gslib.exceptionr   gslib.name_expansionr   r   gslib.progress_callbackr   gslib.storage_urlr   "gslib.third_party.storage_apitoolsr   r   gslib.thread_messager   gslib.utils.cloud_api_helperr   gslib.utils.constantsr   r   gslib.utils.encryption_helperr   r   r   r   gslib.utils.shim_utilr   r   gslib.utils.system_utilr   gslib.utils.text_utilr   r   gslib.utilsr    gslib.utils.translation_helperr!   r   r   r   r-   r4   r9   objectr;   rI   r   r?   r.   r,   <module>r     s    P & %  ' 
   %  / ! 2 ( , 6 ? ? 2 W , < ( & 7 A A = 3 2 1 @ 7 ! C!# 	qs lD"f "r<W r<j/r.   