
    }                     P   S r SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSKrSSK	J
r
  SSKJr  SSKJr  SS	KJr  S
 r\R"                  " SS\S9SS j5       r\R"                  " SS\S9S 5       r " S S\R(                  5      r " S S5      rS r  SS jrS rS r SS jrg)zJFor managing the copy manifest feature (manifest = a file with copy info).    )absolute_import)division)unicode_literalsN)thread_messages)
properties)files)retryc                     AAAU [         :H  $ )z,Check if the exception is a PermissionError.)PermissionError)exc_type	exc_valueexc_tracebackstates       7lib/googlecloudsdk/command_lib/storage/manifest_util.py!_should_retry_if_permission_errorr       s     	_	$$       iX  )max_retrialssleep_msshould_retry_ifc                 ,    [         R                  " XUS9$ )a  Returns the file handle for the given file path.

We use a retry approach here to avoid failing early if
another process, like an antivirus, has acquired the file.
See https://github.com/python/cpython/issues/136965 for more details.

Args:
  file_path (str): The path to the file to open.
  append (bool): Whether to open the file in append mode.
  newline (str|None): The line ending style to use, or None to use plaform
    default.
appendnewline)r   
FileWriter)	file_pathr   r   s      r   get_file_write_handler   (   s    $ 
		)G	DDr   c                 .    [         R                  " U 5      $ )a"  Returns the file handle for the given file path.

We use a retry approach here to avoid failing early if
another process, like an antivirus, has acquired the file.
See https://github.com/python/cpython/issues/136965 for more details.

Args:
  file_path (str): The path to the file to open.
)r   
FileReader)r   s    r   get_file_read_handler    =   s     
		)	$$r   c                        \ rS rSrSrSrSrSrg)ResultStatusO   errorOKskip N)__name__
__module____qualname____firstlineno__ERRORr%   SKIP__static_attributes__r'   r   r   r"   r"   O   s    
%"	$r   r"   c                   (    \ rS rSrSrS rSS jrSrg)ManifestManagerU   z*Handles writing copy statuses to manifest.c                    / SQ[         R                  R                  R                  R	                  5       (       a  S/O/ -   / SQ-   U l        Xl        [        R                  R                  U5      (       a$  [        R                  R                  U5      S:  a  g[        USS9 n[        R                  " X R
                  5      R                  5         SSS5        g! , (       d  f       g= f)z+Creates manifest file with correct headers.)SourceDestinationStartEndMd5UploadId)Source SizeBytes TransferredResultDescriptionr   N
)r   )r   VALUESstoragerun_by_gsutil_shimGetBool_manifest_column_headers_manifest_pathospathexistsgetsizer   csv
DictWriterwriteheader)selfmanifest_pathfile_writers      r   __init__ManifestManager.__init__X   s    	
   ((;;CCEE L	

	
 	!* (	ww~~m$$)G!)K	t
		nn["?"?@LLN
 
 
s   $/C
C*Nc           
      j   U(       a*  UR                   [        R                  L a  UR                  nOSnUR                  R                  S5      nU(       a  UR                  R                  S5      nOUnUR                  (       a-  UR                  R                  SS5      R                  SS5      nOSnUR                  R                  UR                  R                  UUUR                  =(       d    SUR                  UUR                   R                  US.	n[         R"                  R$                  R&                  R)                  5       (       a  S	US
'   [+        U R,                  SSS9 n[.        R0                  " UU R2                  5      R5                  U5        S	S	S	5        g	! , (       d  f       g	= f)zWrites data to manifest file.r   z%Y-%m-%dT%H:%M:%S.%fZr=   z\nz\r )	r3   r4   r5   r6   r7   r9   r:   r;   r<   Nr8   Tr   )result_statusr"   r%   total_bytes_copiedend_timestrftime
start_timedescriptionreplace
source_url
url_stringdestination_urlversionless_url_stringmd5_hashsizevaluer   r>   r?   r@   rA   r   rC   rH   rI   rB   writerow)	rK   manifest_messagefile_progressbytes_copiedrU   rW   rX   row_dictionaryrM   s	            r   	write_rowManifestManager.write_rowy   sd   )77<??J"55ll((112IJH ++445LMjj##$0088uEMM
k k #--88'77NN((.B',,)"0066"
N   33;;==#'nZ 	D$
8;F	nn[2244<H^4L
8 
8 
8s   *1F$$
F2)rB   rC   N)r(   r)   r*   r+   __doc__rN   rf   r.   r'   r   r   r0   r0   U   s    2OB#Mr   r0   c                    U (       a$  [         R                  R                  U 5      (       d
  [        5       $ [        5       n[	        U 5       n[
        R                  " U5      nU HT  nUS   [        R                  R                  [        R                  R                  4;   d  M@  UR                  US   5        MV     SSS5        U$ ! , (       d  f       U$ = f)z>Extracts set of completed or skipped copies from manifest CSV.r;   r3   N)rD   rE   rF   setr    rH   
DictReaderr"   r%   r`   r-   add)rL   resfile_reader
csv_readerrows        r   parse_for_completed_sourcesrr      s    
BGGNN=995L#M*k,J	X<??00,2C2C2I2IJ	JH  +
 
* +*
 
*s   AC$C
Cc                     U R                  [        R                  " UR                  UR                  [        R                  R                  5       UR                  UUUS95        g)z9Send ManifestMessage to task_status_queue for processing.)rZ   r\   rU   r_   rS   r^   rX   N)putr   ManifestMessagestorage_urldatetimeutcnowr_   )task_status_queuesource_resourcedestination_resourcerS   r^   rX   s         r   _send_manifest_messager|      sX     %%$00.::$$++-##% !	r   c           
      N    [        U UU[        R                  S[        U5      S9  g)z;Send ManifestMessage for failed copy to central processing.Nr^   rX   )r|   r"   r,   str)ry   rz   r{   r$   s       r   send_error_messager      s(     e*r   c           	      <    [        U UU[        R                  SUS9  g)z<Send ManifestMessage for skipped copy to central processing.Nr~   )r|   r"   r-   )ry   rz   r{   messages       r   send_skip_messager      s$     r   c                 <    [        XU[        R                  U5        g)z?Send ManifestMessage for successful copy to central processing.N)r|   r"   r%   )ry   rz   r{   r^   s       r   send_success_messager      s    
 *-|Jr   )FN)NNrh   )ri   
__future__r   r   r   rH   rw   enumrD   "googlecloudsdk.command_lib.storager   googlecloudsdk.corer   googlecloudsdk.core.utilr   r	   r   RetryOnExceptionr   r    Enumr"   r0   rr   r|   r   r   r   r'   r   r   <module>r      s    Q &  ' 
   	 > * * *% 5
E
E  5

%

%499 GM GMT
" %)'+	,		 #'Jr   