
    F                         S r SSKrSSKrSSKrSSKrSSKrSSKrSSKJr  S r	S r
S rS rS rS	 rS
 rS rS rS rS rS rg)zSome file reading utilities.
    N)gcsioc                 4   U R                  S5      (       a  g[        R                  R                  U 5      (       a  g[        R                  R	                  U 5      (       a  [        SU -  5      e [        R                  " U 5        g!   [        SU -  5      e= f)zCreates a local directory.

Does nothing if a Google Cloud Storage path is passed.

Args:
  path: the path to create.

Raises:
  ValueError: if path is a file or os.makedir fails.
gs://Nz5Unable to create location. "%s" exists and is a file.zUnable to create location. "%s")
startswithospathisdirisfile
ValueErrormakedirsr   s    -lib/third_party/ml_sdk/cloud/ml/util/_file.pycreate_directoryr      s     
__W
WW]]4
WW^^D
L  ?KK?
6=
>>s   0B Bc                 ^   U R                  S5      (       a%   [        R                  " 5       R                  X5      $ [        X5      $ ! [         a[  n[
        R                  " SU5        [        R                  " S5        [        R                  " 5       R                  X5      s SnA$ SnAff = f)zOpens the given path.r   z-Retrying after exception reading gcs file: %s
   N)	r   r   GcsIOopen	Exceptionloggingerrortimesleep)r   modees      r   open_local_or_gcsr   7   s}    	__W,[[]++   ,mmCQG
jjn[[]++	,s   #A 
B,AB'!B,'B,c                     U R                  S5      (       a$  [        R                  " 5       R                  U 5      $ [        R
                  R                  U 5      $ )z Returns whether the file exists.r   )r   r   r   existsr   r   r   s    r   file_existsr   E   s;    	__W;;=%%77>>$    c                     [        U S5       n[        US5       n[        R                  " X#5        SSS5        SSS5        g! , (       d  f       N= f! , (       d  f       g= f)z}Copy a file from src to dest.

Supports local and Google Cloud Storage.

Args:
  src: source path.
  dest: destination path.
rwN)r   shutilcopyfileobj)srcdesth_srch_dests       r   	copy_filer)   M   sG     c"e	4	%' 
& #"	%	% #"s!   AAA
A	A
A!c                 r    [        U S5       nUR                  U5        SSS5        g! , (       d  f       g= f)zWrites data to a file.

Supports local and Google Cloud Storage.

Args:
  path: output file path.
  data: data to write to file.
r"   N)r   write)r   datar(   s      r   
write_filer-   [   s(     s#v
LL $##s   (
6c                     U R                  S5      (       a  [        U 5      nO[        U 5      nUc  [        SU -  5      eU$ )Nr   zFile cannot be loaded from %s)r   _read_cloud_file_read_local_filer   )r   contents     r   	load_filer2   h   s@    	__Wt$Gt$G_
4t;
<<	.r   c                     U R                  S5      (       a$  [        R                  " 5       R                  U 5      $ [        R                  " U 5      $ Nr   )r   r   r   globr   s    r   
glob_filesr6   t   s6    	__W;;=d##99T?r   c                 p    [        U S5       nUR                  5       sS S S 5        $ ! , (       d  f       g = fNr!   )r   read)
local_pathfs     r   r0   r0   {   s#    J668    '
5c                 p    [        U S5       nUR                  5       sS S S 5        $ ! , (       d  f       g = fr8   )r   r9   )storage_pathr%   s     r   r/   r/      s#    s+s88: ,++r<   c              #      #    [        U [        5      (       d  U OU / HG  nUR                  S5      (       a  [        U5       H  nUv   M	     M1  [	        U5       H  nUv   M	     MI     g 7fr4   )
isinstance
basestringr   _read_cloud_file_stream_read_local_file_stream)	file_listr   lines      r   read_file_streamrF      sd     ))Z@@iG d w)$/$
 0 *$/$
 0s   A'A)c              #   p   #    [        U 5       nU H  nUv   M	     S S S 5        g ! , (       d  f       g = f7f)N)r   )r   file_inrE   s      r   rC   rC      s&     DzWj  zzs   6%	6
36c              #   "  #    [         R                  " SSU S/[         R                  [         R                  S9nUR                   nU H  nUv   M	     S S S 5        UR	                  5       S:w  a  [        SU -  5      eg ! , (       d  f       N1= f7f)Ngsutilcp-)stdoutstderrr   z3Unable to read data from Google Cloud Storage: "%s")
subprocessPopenPIPErM   waitIOError)r   	read_filerH   rE   s       r   rB   rB      s     tS!____) 7j   ^^
G$N
OO  s   ABA>+B>
BB)__doc__r5   r   r   r#   rO   r   apache_beam.ior   r   r   r   r)   r-   r2   r6   r0   r/   rF   rC   rB    r   r   <module>rX      s^      	     ?2 (
	

		Pr   