
    ,                         S r SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJ	r	  SSK
r
SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr   " S S\R,                  5      rg)z"The BigQuery CLI truncate command.    )absolute_import)division)print_function)Optional)app)flagsN)
client_job)client_table)utils)bigquery_command)bq_cached_client)bq_error)bq_id_utils)bq_processor_utilsc                   z  ^  \ rS rSrSrS\S\R                  4U 4S jjrSS\S\	\
   4S jjrS	\R                  R                  4S
 jrS\R                  R                   4S jrS\	\
   4S jrS\S\
4S jrS\R                  R                   S\S\4S jrS\R                  R                   S\S\S\4S jrSrU =r$ )Truncate   zpbq truncate project_id:dataset[.table] [--timestamp] [--dry_run] [--overwrite] [--skip_fully_replicated_tables]
namefvc                 
  > [         [        U ]  X5        [        R                  " SS SSUS9  [        R
                  " SS SUS9  [        R
                  " SS	S
US9  [        R
                  " SSSSUS9  U R                  U5        g )N	timestampz^Optional timestamp to which table(s) will be truncated. Specified as milliseconds since epoch.t)
short_nameflag_valuesdry_runzlNo-op that simply prints out information and the recommended timestamp without modifying tables or datasets.)r   	overwriteFzZOverwrite existing tables. Otherwise timestamp will be appended to all output table names.skip_fully_replicated_tablesTzSkip tables that are fully replicated (synced) and do not need to be truncated back to a point in time. This could result in datasets that have tables synchronized to different points in time, but will require less data to be re-loadeds)superr   __init__r   DEFINE_integerDEFINE_boolean_ProcessCommandRc)selfr   r   	__class__s      (platform/bq/frontend/command_truncate.pyr    Truncate.__init__    s    	(D"4,		$ 
	: 
	" 
&	, 	 	2    
identifierreturnc                   ^
 [         R                  R                  5       m
U(       a$  [        R                  " T
UR                  5       S9nO[        R                  " S5      eSU l        SU l	        SU l
        / nU R                  (       a  U R                  (       d  [        SU R                  -  5        [        U[        R                   R"                  5      (       a  U/nO`[        U[        R                   R$                  5      (       a7  ['        [)        U
4S j[*        R,                  " T
R.                  USS95      5      nW H9  n UR1                  U R3                  U[5        U R                  5      S5      5        M;     GO`[        U[        R                   R"                  5      (       a  U R=                  U5      nO:[        U[        R                   R$                  5      (       a  U R?                  U5      n [A        ['        [)        U RB                  W5      5      5      nU(       d  [        R                  " SU-  5      e[        SU-  5        W H  n[I        US5      (       d  [K        S5      e [        R                   R"                  RM                  URN                  URP                  US   S9n	UR1                  U R3                  U	[5        U5      US   5      5        M     [        U R                  < SU R                  < SU R                  < S35        [        USS06  g! [6        R8                   aM  n[        U5        UR1                  U R;                  US	5      5        U =R                  S
-  sl
         SnAGM<  SnAff = f! [D        [6        RF                  4 a    Sn GNf = f! [6        R8                   aM  n[        U5        UR1                  U R;                  W	S	5      5        U =R                  S
-  sl
         SnAGM  SnAff = f)zTruncates table/dataset/project to a particular timestamp.

Examples:
  bq truncate project_id:dataset
  bq truncate --overwrite project_id:dataset --timestamp 123456789
  bq truncate --skip_fully_replicated_tables=false project_id:dataset
id_fallbacksr)   z-Must specify one of project, dataset or tabler   zQTruncating to user specified timestamp %s.(Not skipping fully replicated tables.)c                 4   > [         R                  " TU S   S9$ )Nidr,   )bq_client_utilsGetReference)xclients    r&   <lambda>&Truncate.RunWithArgs.<locals>.<lambda>g   s    O88#)agr(   @B )	apiclient	referencemax_resultsFFailed   Nz:Unable to figure out a recovery timestamp for %s. Exiting.z*Recommended timestamp to truncate to is %s	datasetIdz!Missing `datasetId` on reference.r   	projectIdr<   tableIdfully_replicatedz tables truncated, z tables failed to truncate, z tables skippedsep
))r   ClientGetr0   r1   stripr   
UsageErrortruncated_table_countskipped_table_countfailed_table_countr   r   print
isinstancer   ApiClientHelperTableReferenceDatasetReferencelistmapr
   list_tablesr7   append_TruncateTablestrr   BigqueryError_formatOutputString_GetTableInfo_GetTableInfosFromDatasetmin_GetRecoveryTimestamp
ValueErrorBigqueryTypeErrorhasattrAttributeErrorCreater>   r<   )r$   r)   r8   status
all_tablesa_tableeall_table_infosrecovery_timestamptable_referencer3   s             @r&   RunWithArgsTruncate.RunWithArgsE   sk    $$((*F!..**:*:*<i NNJKK!"D DDF~~dll!NN
 
I{::II	J	J[
i!<!<!M!MNN ** & 0 0 )"-		*  '	'
--!!'3t~~+>F   
I{::II	J	J,,Y7i!<!<!M!MNN ::9E/" T//AB
  nnH
 	
 8;MMN$'y+..BC
C	''77FFMM!++!++fo N /
 --!!!(),- %( 
 &&##$$		
 
6tq %% 	'
(
--11'8D
F

!
!Q
&
!
!	' (445 "!"< %% 	'
(
--11/8L
N

!
!Q
&
!
!	'sE   5L!6(N .A.N(!N5AM==NN%$N%(P	<APP	dataset_referencec                 D    SUR                   -  nU R                  US5      $ )Na}  SELECT
  TABLE_NAME,
  UNIX_MILLIS(replicated_time_at_remote_site),
  CASE
    WHEN last_update_time <= min_latest_replicated_time THEN TRUE
  ELSE
  FALSE
END
  AS fully_replicated
FROM (
  SELECT
    TABLE_NAME,
    multi_site_info.last_update_time,
    ARRAY_AGG(site_info.latest_replicated_time
    ORDER BY
      latest_replicated_time DESC)[safe_OFFSET(1)] AS replicated_time_at_remote_site,
    ARRAY_AGG(site_info.latest_replicated_time
    ORDER BY
      latest_replicated_time ASC)[safe_OFFSET(0)] AS min_latest_replicated_time
  FROM
    %s.INFORMATION_SCHEMA.TABLES t,
    t.multi_site_info.site_info
  GROUP BY
    1,
    2)r6   )r<   _ReadTableInfo)r$   ri   $recovery_timestamp_for_dataset_querys      r&   rX   "Truncate._GetTableInfosFromDataset   s4    -
0 ((1,)(2 ,k r(   rf   c                 `    SUR                   < SUR                  < S3nU R                  USS9$ )Na%  SELECT
  TABLE_NAME,
  UNIX_MILLIS(replicated_time_at_remote_site),
  CASE
    WHEN last_update_time <= min_latest_replicated_time THEN TRUE
  ELSE
  FALSE
END
  AS fully_replicated
FROM (
  SELECT
    TABLE_NAME,
    multi_site_info.last_update_time,
    ARRAY_AGG(site_info.latest_replicated_time
    ORDER BY
      latest_replicated_time DESC)[safe_OFFSET(1)] AS replicated_time_at_remote_site,
    ARRAY_AGG(site_info.latest_replicated_time
    ORDER BY
      latest_replicated_time ASC)[safe_OFFSET(0)] AS min_latest_replicated_time
  FROM
    zX.INFORMATION_SCHEMA.TABLES t,
    t.multi_site_info.site_info
  WHERE
    TABLE_NAME = 'z'
  GROUP BY
    1,
    2 )r;   )	row_count)r<   r?   rk   )r$   rf   "recovery_timestamp_for_table_querys      r&   rW   Truncate._GetTableInfo   s8    @ ((/*A*A5*C&6 AQOOr(   c                 6    US   (       a  [        US   5      $ S $ )Nre   )int)r$   
table_infos     r&   rZ   Truncate._GetRecoveryTimestamp   s,     *+ 	J+,- r(   queryro   c                    [         R                  R                  5       n [        R                  " X1SS9n/ n[        R                  " U5      (       dl  [        R                  " X4S   SUS9u  px[        [        U5      5       H7  n	0 n
X   S   U
S	'   X   S
   U
S'   X   S   S:H  U
S'   UR                  U
5        M9     U$ g ! [
        R                   a0  nSUR                  S   ;   a  [        R                  " S5      eUeS nAff = f)NF)use_legacy_sqlzName multi_site_info not foundmessagez:This functionality is not enabled for the current project.jobReferencer   )	start_rowmax_rowsr   r;   re      truer@   )r   rC   rD   r	   Queryr   rU   errorr   rF   r0   IsFailedJobReadSchemaAndJobRowsrangelenrR   )r$   rv   ro   r3   jobrc   rd   _rowsirt   s              r&   rk   Truncate._ReadTableInfo   s   $$((*FV5Ac O&&s++//
n%Yga SY!
!WQZ
6+/71:
'()-v)=
%&z*    , !! 	 
*QWWY-?	?nnH
 	
 	s   B? ?D+C>>Dr`   c                     SX4-  $ )Nz%s %200s )r$   rf   r`   s      r&   rV   Truncate._formatOutputString  s    
 111r(   re   is_fully_replicatedc           	      Z   [         R                  R                  5       n0 nU R                  (       dZ  [        R
                  R                  R                  UR                  UR                  SR                  UR                  SU/5      S9nOUnU R                  (       a.  U(       a'  U =R                  S-  sl        U R                  US5      $ U R                  (       a  U R                  USU-  5      $ SSS	S
.n[         R"                  R$                  (       a  [         R"                  R$                  US'   [&        R(                  " UU< SU< 3S9nSn [*        R,                  " XG/U40 UD6n	U	c'  U =R.                  S-  sl        U R                  US5      $ [0        R2                  " U	5      nU =R4                  S-  sl        U R                  USU-  5      $ ! [6        R8                   a?  n
[;        U
5        U =R.                  S-  sl        U R                  USU-  5      s S n
A
$ S n
A
ff = f)Nr   TRUNCATED_ATr=   r;   zFully replicated...Skippedzwill be Truncated@%sWRITE_TRUNCATEFCOPY)write_dispositionignore_already_existsoperation_typelocation@r,    r:   zSuccessful %s z
Failed %s )r   rC   rD   r   r   rL   rM   r_   r>   r<   joinr?   r   rH   rV   r   bq_flagsLOCATIONvaluer0   GetTableReferencer	   	CopyTablerI   r   ConstructObjectReferencerG   r   rU   rJ   )r$   rf   re   r   r3   kwdsdestsource_tablejob_refr   rc   s              r&   rS   Truncate._TruncateTable  s    $$((*FD>>((77>>#--#--((&&8JK ? d d((-@
!#%%
7  ||%%
&);;  .!& D
 !**00d:"44-/ABL GD  FFc	1$''h77";;C@g
  A% %%d,<w,FGG!! DAh
"%%dL7,BCCDs%   AG ?G H*+4H%H*%H*)rI   rH   rG   ) )__name__
__module____qualname____firstlineno__usagerT   r   
FlagValuesr    r   rs   rg   r   rL   rN   rX   rM   rW   rZ   rk   rV   boolrS   __static_attributes____classcell__)r%   s   @r&   r   r      s   %#3 #E$4$4 #JiC i# iV!*::KK!F!P(88GG!PF# # # 62"22AA2 2 	23D"22AA3D 3D  	3D
 3D 3Dr(   r   )__doc__
__future__r   r   r   typingr   abslr   r   r   clientsr	   r
   r   r0   frontendr   r   r   r   r   BigqueryCmdr   r   r(   r&   <module>r      sL    ( &  %        , % %   $vD++ vDr(   