
    Y<                        S r SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSKrSSK	r	SSK
Jr  SSK
Jr  SSK
Jr  SS	K
Jr  SS
KJr  SSKJr  SSKJr  SSKrSr " S S\R.                  5      r " S S\R.                  5      r\R4                  " SSS/5      r " S S5      rS r " S S\R<                  " \R>                  \ 5      5      r! " S S\!5      r" " S S\!\RF                  5      r$S r%S$S jr& " S  S!5      r' " S" S#5      r(g)%z1Tools for monitoring and reporting task statuses.    )absolute_import)division)unicode_literalsN)errors)manifest_util)metrics_util)thread_messages)log)progress_tracker)scaled_integer   c                   $    \ rS rSrSrSrSrSrSrg)OperationName*   DownloadingzIntra-Cloud CopyingzDaisy Chain Copying	Uploading N)	__name__
__module____qualname____firstlineno__DOWNLOADINGINTRA_CLOUD_COPYINGDAISY_CHAIN_COPYING	UPLOADING__static_attributes__r       ;lib/googlecloudsdk/command_lib/storage/tasks/task_status.pyr   r   *   s    +--)r   r   c                       \ rS rSrSrSrSrg)IncrementType1   INTEGERFILES_AND_BYTESr   N)r   r   r   r   r"   r#   r   r   r   r   r    r    1   s    '%/r   r    ProgressManagerArgsincrement_typemanifest_pathc                   (    \ rS rSrSr   SS jrSrg)FileProgress:   aJ  Holds progress information for file being copied.

Attributes:
  component_progress (dict<int,int>): Records bytes copied per component. If
    not multi-component copy (e.g. "sliced download"), there will only be one
    component.
  start_time (datetime|None): Needed if writing file copy results to manifest.
  total_bytes_copied (int|None): Sum of bytes copied for each component.
    Needed because components are popped when completed, but we don't want to
    lose info on them if writing to the manifest.
  error_occurred (bool): Whether an error occurred during the operation.
Nc                 t    [        U5       Vs0 s H  oUS_M     snU l        X l        X0l        X@l        g s  snf Nr   )rangecomponent_progress
start_timetotal_bytes_copiederror_occurred)selfcomponent_countr.   r/   r0   is         r   __init__FileProgress.__init__H   s;     .3?-CD-C!t-CDD O0( Es   5)r-   r0   r.   r/   )NNF)r   r   r   r   __doc__r4   r   r   r   r   r(   r(   :   s      
)r   r(   c                 N    [        X-  S5      n[        R                  " USS9S-   $ )Nr      decimal_placesz/s)maxr   FormatBinaryNumber)bytes_processed
time_deltathroughput_bytess      r   _get_formatted_throughputr@   U   s2    5q9		*	*q
*,0
1 1r   c                   r    \ rS rSrSr\R                  S 5       rS r\R                  S 5       r	S r
S rSrg	)
_StatusTracker[   z>Abstract class for tracking and displaying operation progress.c                     g)z4Generates string to illustrate progress to the user.Nr   r1   s    r   _get_status_string!_StatusTracker._get_status_string^   s     	r   c                     g)z.Generates string for when StatusTracker exits.
r   rE   s    r   _get_done_string_StatusTracker._get_done_stringc   s    r   c                     g)z}Processes task status message for printing and aggregation.

Args:
  status_message (thread_messages.*): Message to process.
Nr   r1   status_messages     r   add_message_StatusTracker.add_messageg   s     	r   c                     [         R                  " SU R                  U R                  SS9U l        U R                  R                  5         U $ )Nz  T)messagedetail_message_callbackdone_message_callback
no_spacing)r   ProgressTrackerrF   rJ   _progress_tracker	__enter__rE   s    r   start_StatusTracker.startp   sF    -== $ 7 7"33	D
 	$$&Kr   c                 `    U R                   (       a  U R                   R                  XU5        g g N)rW   __exit__r1   exc_typeexc_valexc_tbs       r   stop_StatusTracker.stopy   s&    
%%h@ r   )rW   N)r   r   r   r   r6   abcabstractmethodrF   rJ   rO   rY   rb   r   r   r   r   rB   rB   [   sG    F	 	 	 	Ar   rB   c                   8   ^  \ rS rSrSrU 4S jrS rS rSrU =r	$ )_IntegerStatusTracker~   8See super class. Tracks both file count and byte amount.c                 F   > [         [        U ]  5         SU l        SU l        g r+   )superrg   r4   
_completed_total_estimation)r1   	__class__s    r   r4   _IntegerStatusTracker.__init__   s     	
/1DODr   c                     U R                   (       a'  SR                  U R                  U R                   5      nOU R                  nSR                  U5      $ )See super class.{}/{}zCompleted {})rm   formatrl   )r1   file_progress_strings     r   rF   (_IntegerStatusTracker._get_status_string   sH    $^^DOO,0,B,BD "__""#788r   c                     [        U[        R                  5      (       a   U =R                  UR                  -  sl        g[        U[        R
                  5      (       a  U =R                  S-  sl        ggrq   r8   N)
isinstancer	   WorkloadEstimatorMessagerm   
item_countIncrementProgressMessagerl   rM   s     r   rO   !_IntegerStatusTracker.add_message   sS    ./"J"JKK
 9 99	NO$L$L	M	M
ooo 
Nr   )rl   rm   )
r   r   r   r   r6   r4   rF   rO   r   __classcell__rn   s   @r   rg   rg   ~   s    @
9 r   rg   c                   `   ^  \ rS rSrSrSU 4S jjrS rS rS rS r	S r
S	 rU 4S
 jrSrU =r$ )_FilesAndBytesStatusTracker   ri   c                   > [         [        U ]  5         SU l        SU l        SU l        SU l        S U l        S U l        SU l	        S U l
        SU l        S U l        0 U l        U(       a  [        R                  " U5      U l        g S U l        g r+   )rk   r   r4   _completed_files_processed_bytes_total_files_estimation_total_bytes_estimation_first_operation_time_last_operation_time_total_processed_bytes_window_start_time_window_processed_bytes_window_throughput_tracked_file_progressr   ManifestManager_manifest_manager)r1   r&   rn   s     r   r4   $_FilesAndBytesStatusTracker.__init__   s    	
%t57DD#$D #$D  "&D $D"#D #D#$D "D #%D,<<]Kd#dr   c                    [         R                  " U R                  SS9nU R                  (       a'  SR	                  U R
                  U R                  5      nOU R
                  nU R                  (       a2  [         R                  " U R                  SS9nSR	                  UU5      nOUnU R                  (       a  SU R                  -   nOSnSR	                  UUU5      $ )rq   r8   r9   rr   z |  zCompleted files {} | {}{})r   r<   r   r   rs   r   r   r   )r1   scaled_processed_bytesrt   scaled_total_bytes_estimationbytes_progress_stringthroughput_addendum_strings         r   rF   ._FilesAndBytesStatusTracker._get_status_string   s    +>>a1##$^^D,A,A,0,H,HJ "22##&4&G&G

&
&q':#%nn-C-JL 5#(4+B+B#B #% (//0D0E0JL Lr   c                 ^   U R                   c#  UR                  U l         UR                  U l        OUR                  U l        U =R                  U-  sl        UR                  U R                  -
  nU[
        :  a4  [        U R                  U5      U l        UR                  U l        SU l        gg)z<Updates stats and recalculates throughput if past threshold.Nr   )r   timer   r   r   $_THROUGHPUT_WINDOW_THRESHOLD_SECONDSr@   r   )r1   rN   processed_bytesr>   s       r   _update_throughput._FilesAndBytesStatusTracker._update_throughput   s    !!)#1#6#6d  . 3 3d"0"5"5d  O3 $$t'>'>>J88 9

&
&
!4d . 3 3d%&d"	 9r   c                     U =R                   UR                  -  sl         U =R                  UR                  -  sl        g)z?Adds WorloadEstimatorMessage info to total workload estimation.N)r   rz   r   sizerM   s     r   _add_to_workload_estimation7_FilesAndBytesStatusTracker._add_to_workload_estimation   s0      N$=$==   N$7$77 r   c                    UR                   R                  nX R                  ;  a  UR                  (       a!  [	        UR                  S9U R                  U'   O[	        SS9U R                  U'   U R
                  (       ah  [        R                  R                  UR                  [        R                  R                  5      U R                  U   l        SU R                  U   l        U R                  U   R                  nUR                  (       a  UR                  nOSnUR                  UR                   -
  nXSR#                  US5      -
  nU =R$                  U-  sl        U R'                  X5        U R
                  (       a"  U R                  U   =R                  U-  sl        UR(                  (       a  SU R                  U   l        XQR*                  :X  am  UR-                  US5        U(       dS  U R                  U   R(                  (       d  U =R.                  S-  sl        U R
                  (       d  U R                  U	 gggXSU'   g)z-Track progress of a multipart file operation.)r2   r8   r   TN)
source_url
url_stringr   total_componentsr(   r   datetimefromtimestampr   timezoneutcr.   r/   r-   component_numbercurrent_byteoffsetgetr   r   r0   lengthpopr   )r1   rN   file_url_stringcomponent_trackerr   processed_component_bytesnewly_processed_bytess          r   _add_progress)_FilesAndBytesStatusTracker._add_progress   s   $//::O999		(	(7C*;;8=##O4 8D8##O4			++N,?,?,4,=,=,A,AC 	##O4? KL##O4G33++  &&'88 	##n&;&;;  	"$9$9:JA$NN 22NB
!!
 .--1FG-$$DHd!!/2A $9$99,d3**?;JJ


1
$
%%))/: &	  -F()r   c                     U R                   (       d  [        R                  " S5      eU R                  R	                  UR
                  R                  S5      nU R                   R                  X5        g)z<Updates manifest file and pops file from tracking if needed.zRReceived ManifestMessage but StatusTracker was not initialized with manifest path.N)r   r   Errorr   r   r   r   	write_row)r1   rN   file_progresss      r   _add_to_manifest,_FilesAndBytesStatusTracker._add_to_manifest  s^    !!LL  //33!!,,d4M$$^Cr   c                    [        U[        R                  5      (       a  U R                  U5        g[        U[        R                  5      (       a#  U R                  U5        U R                  U5        g[        U[        R                  5      (       a  U =R                  S-  sl        g[        U[        R                  5      (       a  U R                  U5        ggrw   )rx   r	   ry   r   DetailedProgressMessage#_set_source_and_destination_schemesr   r{   r   ManifestMessager   rM   s     r   rO   '_FilesAndBytesStatusTracker.add_message'  s    ./"J"JKK
&&~6	NO$K$K	L	L
..~>
(	NO$L$L	M	M
q 	NO$C$C	D	D
N+ 
Er   c                   > [         [        U ]  XU5        U R                  b  U R                  b  U R                  U R                  :w  a  U R                  U R                  -
  n[
        R                  R                  SR                  [        U R                  U5      5      5        U R                  U R                  UU R                  5        g g g g )Nz
Average throughput: {})rk   r   rb   r   r   r
   statusPrintrs   r@   r   _report_metricsr   )r1   r_   r`   ra   r>   rn   s        r   rb    _FilesAndBytesStatusTracker.stop4  s    	
%t1(VL"".!!-""d&?&??,,t/I/IIj	jj188
#D$9$9:
FH I 400*002 	@ 	. 	/r   )r   r   r   r   r   r   r   r   r   r   r   r   r\   )r   r   r   r   r6   r4   rF   r   r   r   r   rO   rb   r   r}   r~   s   @r   r   r      s7    @$6L4'"8
4Fl	D,2 2r   r   c                     Sn U R                  5       nUS:X  a  OU(       a  UR                  U5        OSnM4  U(       a  [        R                  " S5        gg)zGThread method for submiting items from queue to tracker for processing.FT	_SHUTDOWNzLStatus message submitted to task_status_queue without a manager to print it.N)r   rO   r
   warning)task_status_queuestatus_trackerunhandled_message_existsrN   s       r   status_message_handlerr   C  s[    "&**,N$  0!% 	 KK ( ) r   c                 2    U b  [        X5      $ [        5       $ )aZ  Factory function that returns a ProgressManager instance.

Args:
  task_status_queue (multiprocessing.Queue|None): Tasks can submit their
    progress messages here.
  progress_manager_args (ProgressManagerArgs|None): Determines what type of
    progress indicator to display.

Returns:
  An instance of _ProgressManager or _NoOpProgressManager.
)_ProgressManager_NoOpProgressManager)r   progress_manager_argss     r   progress_managerr   U  s     "-EE!!r   c                   .    \ rS rSrSrSS jrS rS rSrg)	r   ig  zContext manager for processing and displaying progress completing command.

Ensure that this class is instantiated after all the child
processes (if any) are started to prevent deadlock.
Nc                 8    X l         SU l        SU l        Xl        g)zInitializes context manager.

Args:
  task_status_queue (multiprocessing.Queue): Tasks can submit their progress
    messages here.
  progress_manager_args (ProgressManagerArgs|None): Determines what type of
    progress indicator to display.
N_progress_manager_args_status_message_handler_thread_status_tracker_task_status_queue)r1   r   r   s      r   r4   _ProgressManager.__init__n  s      #8*.D'D/r   c                    U R                   (       a  U R                   R                  [        R                  L a  [	        5       U l        OKU R                   R                  [        R                  L a$  [        U R                   R                  5      U l        [        R                  " [        U R                  U R
                  4S9U l        U R                  R                  5         U R
                  (       a  U R
                  R                  5         U $ )N)targetargs)r   r%   r    r"   rg   r   r#   r   r&   	threadingThreadr   r   r   rY   rE   s    r   rX   _ProgressManager.__enter__|  s    ""		$	$	3	3}7L7L	L46''66))*:''55 7 +4*:*:%%%t';';<+>D' 	''--/
  "Kr   c                     U R                   R                  S5        U R                  R                  5         U R                  (       a  U R                  R                  XU5        g g )Nr   )r   putr   joinr   rb   r^   s       r   r]   _ProgressManager.__exit__  sL    ,'',,.
6: r   r   r\   )	r   r   r   r   r6   r4   rX   r]   r   r   r   r   r   r   g  s    0$;r   r   c                   $    \ rS rSrSrS rS rSrg)r   i  zyProgress Manager that does not do anything.

Similar to contextlib.nullcontext, but it is available only for Python3.7+.
c                     U $ r\   r   rE   s    r   rX   _NoOpProgressManager.__enter__  s    Kr   c                 
    AAAg r\   r   r^   s       r   r]   _NoOpProgressManager.__exit__  s    	7Fr   r   N)r   r   r   r   r6   rX   r]   r   r   r   r   r   r     s    
	r   r   )NN))r6   
__future__r   r   r   rd   collectionsr   enumr   "googlecloudsdk.command_lib.storager   r   r   r	   googlecloudsdk.corer
   googlecloudsdk.core.consoler   googlecloudsdk.core.utilr   sixr   Enumr   r    
namedtupler$   r(   r@   with_metaclassABCMetaobjectrB   rg   MetricsReporterr   r   r   r   r   r   r   r   <module>r      s    8 &  ' 
     5 < ; > # 8 3 

 () $DII &DII &
 ",,,o>@ ) )61 AS''V<  AFN 4h2.,2N2N h2V)$"$,; ,;^	 	r   