
    >                       S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
Jr  SSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKrSSKJr  SS	KJr  SSKrSS
KJr  SSKJr  SSKJr  SSKJ r   SSKJ!r!  SSK"J#r#  SSK$J%r%  SSK&J'r'  SSK(J)r)  SSK*J+r+  SSK,J-r-  SSK.J/r/  SSK0J1r1  SSK0J2r2  SSK3J4r4  SSK3J5r5  SSK3J6r6  SSK7J8r9  SSK:J;r;  SSK:J<r<  SSK:J=r=  SSK>J?r?  SS K@JArA  SS!KBJCrC  SS"KBJDrD  SS#KBJErE  SS$KBJFrF  SS%KGJHrH  SS&KGJIrI  SS'KGJJrJ  SS(KGJKrK  SS)KLJMrM  SS*KLJNrN  SS+KLJOrO  SS,KPJQrQ  SS-KPJRrR  SS.KPJSrS  SS/KPJTrT  SS0KPJUrU  SS1KPJVrV  SS2KPJWrW  SS3KPJXrX  SS4KPJYrY  SS5KPJZrZ  SS6KPJ[r[  SS7KPJ\r\  SS8KPJ]r]  SS9KPJ^r^  SS:KPJ_r_  SS;KPJ`r`  SS<KPJara  SS=KPJbrb  SS>KPJcrc  SS?KPJdrd  SS@KeJfrf  SSAKeJgrg  SSBKhJiri  SSCKhJjrj  SSDKkJlrl  SSEKmJnrn  SSFKoJprp  SSGKoJqrq  SSHKoJrrr  SSIKsJtrt  \R                  (       a  \vrwSJrxSK\x-   SL-   rySMrzSNr{SOr|/ r}SP r~SQ rSR rSS rSfST jrSU rSV rSW rSX rSY rSZ r " S[ S\\5      r " S] S^\5      r " S_ S`\5      rSfSa jrSb rSc r " Sd Se\ 5      rg)gz*Implementation of Unix-like rsync command.    )absolute_import)print_function)division)unicode_literalsN)islice)urllib)config)BucketListingObject)NotFoundException)ServiceException)Command)DummyArgChecker)'ShimTranslatePredefinedAclSubOptForCopy)CommandArgument)ApiSelector)CommandException)LogPerformanceSummaryParams)PluralityCheckableIterator)SeekAheadResult)GetCaughtSignals)RegisterSignalHandler)GenerationFromUrlAndString)IsCloudSubdirPlaceholder)StorageUrlFromString)storage_v1_messages)	constants)copy_helper)parallelism_framework_util)UsingCrcmodExtension)GetCloudApiInstance)CreateCopyHelperOpts)GetSourceFieldsNeededForCopy)GZIP_ALL_FILES)SkipUnsupportedObjectError)%CalculateB64EncodedCrc32cFromContents)"CalculateB64EncodedMd5FromContents)SLOW_CRCMOD_RSYNC_WARNING)SLOW_CRCMOD_WARNING)CreateCustomMetadata) GetValueFromObjectCustomMetadata)ObjectIsGzipEncoded)
ATIME_ATTR)ConvertDatetimeToPOSIX)ConvertModeToBase8)+DeserializeFileAttributesFromObjectMetadata)GID_ATTR)InitializePreservePosixData)	MODE_ATTR)
MTIME_ATTR)NA_ID)NA_MODE)NA_TIME)NeedsPOSIXAttributeUpdate)ParseAndSetPOSIXAttributes)POSIXAttributes)'SerializeFileAttributesToObjectMetadata)UID_ATTR)ValidateFilePermissionAccess)WarnFutureTimestamp)WarnInvalidValue)WarnNegativeAttribute)
DiffAction)RsyncDiffToApply)GcloudStorageFlag)GcloudStorageMap)
IS_WINDOWS)CopyCustomMetadata)CalculateThroughput)SECONDS_PER_DAY)TEN_MIB)CreateWildcardIteratorz,
  gsutil rsync [OPTION]... src_url dst_url
z
<B>SYNOPSIS</B>
a	S  


<B>DESCRIPTION</B>
  The gsutil rsync command makes the contents under dst_url the same as the
  contents under src_url, by copying any missing files/objects (or those whose
  data has changed), and (if the -d option is specified) deleting any extra
  files/objects. src_url must specify a directory, bucket, or bucket
  subdirectory. For example, to sync the contents of the local directory "data"
  to the bucket gs://mybucket/data, you could do:

    gsutil rsync data gs://mybucket/data

  To recurse into directories use the -r option:

    gsutil rsync -r data gs://mybucket/data

  If you have a large number of objects to synchronize you might want to use the
  gsutil -m option (see "gsutil help options"), to perform parallel
  (multi-threaded/multi-processing) synchronization:

    gsutil -m rsync -r data gs://mybucket/data

  The -m option typically will provide a large performance boost if either the
  source or destination (or both) is a cloud URL. If both source and
  destination are file URLs the -m option will typically thrash the disk and
  slow synchronization down.

  Note 1: Shells (like bash, zsh) sometimes attempt to expand wildcards in ways
  that can be surprising. Also, attempting to copy files whose names contain
  wildcard characters can result in problems. For more details about these
  issues see `Wildcard behavior considerations
  <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_.

  Note 2: If you are synchronizing a large amount of data between clouds you
  might consider setting up a
  `Google Compute Engine <https://cloud.google.com/products/compute-engine>`_
  account and running gsutil there. Since cross-provider gsutil data transfers
  flow through the machine where gsutil is running, doing this can make your
  transfer run significantly faster than running gsutil on your local
  workstation.

  Note 3: rsync does not copy empty directory trees, since Cloud Storage uses a
  `flat namespace <https://cloud.google.com/storage/docs/folders>`_.


<B>Using -d Option (with caution!) to mirror source and destination.</B>
  The rsync -d option is very useful and commonly used, because it provides a
  means of making the contents of a destination bucket or directory match those
  of a source bucket or directory. This is done by copying all data from the
  source to the destination and deleting all other data in the destination that
  is not in the source. Please exercise caution when you
  use this option: It's possible to delete large amounts of data accidentally
  if, for example, you erroneously reverse source and destination.

  To make the local directory my-data the same as the contents of
  gs://mybucket/data and delete objects in the local directory that are not in
  gs://mybucket/data:

    gsutil rsync -d -r gs://mybucket/data my-data

  To make the contents of gs://mybucket2 the same as gs://mybucket1 and delete
  objects in gs://mybucket2 that are not in gs://mybucket1:

    gsutil rsync -d -r gs://mybucket1 gs://mybucket2

  You can also mirror data across local directories. This example will copy all
  objects from dir1 into dir2 and delete all objects in dir2 which are not in dir1:

    gsutil rsync -d -r dir1 dir2

  To mirror your content across clouds:

    gsutil rsync -d -r gs://my-gs-bucket s3://my-s3-bucket

  Change detection works if the other Cloud provider is using md5 or CRC32. AWS
  multipart upload has an incompatible checksum.

  As mentioned above, using -d can be dangerous because of how quickly data can
  be deleted. For example, if you meant to synchronize a local directory from
  a bucket in the cloud but instead run the command:

    gsutil -m rsync -r -d ./your-dir gs://your-bucket

  and your-dir is currently empty, you will quickly delete all of the objects in
  gs://your-bucket.

  You can also cause large amounts of data to be lost quickly by specifying a
  subdirectory of the destination as the source of an rsync. For example, the
  command:

    gsutil -m rsync -r -d gs://your-bucket/data gs://your-bucket

  would cause most or all of the objects in gs://your-bucket to be deleted
  (some objects may survive if there are any with names that sort lower than
  "data" under gs://your-bucket/data).

  In addition to paying careful attention to the source and destination you
  specify with the rsync command, there are two more safety measures you can
  take when using gsutil rsync -d:

  1. Try running the command with the rsync -n option first, to see what it
     would do without actually performing the operations. For example, if
     you run the command:

       gsutil -m rsync -r -d -n gs://your-bucket/data gs://your-bucket

     it will be immediately evident that running that command without the -n
     option would cause many objects to be deleted.

  2. Enable object versioning in your bucket, which allows you to restore
     objects if you accidentally delete them. For more details see
     `Object Versioning
     <https://cloud.google.com/storage/docs/object-versioning>`_.


<B>BE CAREFUL WHEN SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SYMLINKS, DEVICES, ETC.)</B>
  Running gsutil rsync over a directory containing operating system-specific
  file types (symbolic links, device files, sockets, named pipes, etc.) can
  cause various problems. For example, running a command like:

    gsutil rsync -r ./dir gs://my-bucket

  will cause gsutil to follow any symbolic links in ./dir, creating objects in
  my-bucket containing the data from the files to which the symlinks point. This
  can cause various problems:

  * If you use gsutil rsync as a simple way to backup a directory to a bucket,
    restoring from that bucket will result in files where the symlinks used
    to be. At best this is wasteful of space, and at worst it can result in
    outdated data or broken applications -- depending on what is consuming
    the symlinks.

  * If you use gsutil rsync over directories containing broken symlinks,
    gsutil rsync will abort (unless you pass the -e option).

  * gsutil rsync skips symlinks that point to directories.

  Since gsutil rsync is intended to support data operations (like moving a data
  set to the cloud for computational processing) and it needs to be compatible
  both in the cloud and across common operating systems, there are no plans for
  gsutil rsync to support operating system-specific file types like symlinks.

  We recommend that users do one of the following:

  * Don't use gsutil rsync over directories containing symlinks or other OS-
    specific file types.
  * Use the -e option to exclude symlinks or the -x option to exclude
    OS-specific file types by name.
  * Use a tool (such as tar) that preserves symlinks and other OS-specific file
    types, packaging up directories containing such files before uploading to
    the cloud.


<B>EVENTUAL CONSISTENCY WITH NON-GOOGLE CLOUD PROVIDERS</B>
  While Google Cloud Storage is strongly consistent, some cloud providers
  only support eventual consistency. You may encounter scenarios where rsync
  synchronizes using stale listing data when working with these other cloud
  providers. For example, if you run rsync immediately after uploading an
  object to an eventually consistent cloud provider, the added object may not
  yet appear in the provider's listing. Consequently, rsync will miss adding
  the object to the destination. If this happens you can rerun the rsync
  operation again later (after the object listing has "caught up").



<B>FAILURE HANDLING</B>
  The rsync command retries failures when it is useful to do so, but if
  enough failures happen during a particular copy or delete operation, or if
  a failure isn't retryable, the overall command fails.

  If the -C option is provided, the command instead skips failing objects and
  moves on. At the end of the synchronization run, if any failures were not
  successfully retried, the rsync command reports the count of failures and
  exits with non-zero status. At this point you can run the rsync command
  again, and gsutil attempts any remaining needed copy and/or delete
  operations.

  For more details about gsutil's retry handling, see `Retry strategy
  <https://cloud.google.com/storage/docs/retry-strategy#tools>`_.


<B>CHANGE DETECTION ALGORITHM</B>
  To determine if a file or object has changed, gsutil rsync first checks
  whether the file modification time (mtime) of both the source and destination
  is available. If mtime is available at both source and destination, and the
  destination mtime is different than the source, or if the source and
  destination file size differ, gsutil rsync will update the destination. If the
  source is a cloud bucket and the destination is a local file system, and if
  mtime is not available for the source, gsutil rsync will use the time created
  for the cloud object as a substitute for mtime. Otherwise, if mtime is not
  available for either the source or the destination, gsutil rsync will fall
  back to using checksums. If the source and destination are both cloud buckets
  with checksums available, gsutil rsync will use these hashes instead of mtime.
  However, gsutil rsync will still update mtime at the destination if it is not
  present. If the source and destination have matching checksums and only the
  source has an mtime, gsutil rsync will copy the mtime to the destination. If
  neither mtime nor checksums are available, gsutil rsync will resort to
  comparing file sizes.

  Checksums will not be available when comparing composite Google Cloud Storage
  objects with objects at a cloud provider that does not support CRC32C (which
  is the only checksum available for composite objects). See 'gsutil help
  compose' for details about composite objects.


<B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B>
  If both the source and destination URL are cloud URLs from the same provider,
  gsutil copies data "in the cloud" (i.e., without downloading to and uploading
  from the machine where you run gsutil). In addition to the performance and
  cost advantages of doing this, copying in the cloud preserves metadata (like
  Content-Type and Cache-Control). In contrast, when you download data from the
  cloud it ends up in a file, which has no associated metadata, other than file
  modification time (mtime). Thus, unless you have some way to hold on to or
  re-create that metadata, synchronizing a bucket to a directory in the local
  file system will not retain the metadata other than mtime.

  Note that by default, the gsutil rsync command does not copy the ACLs of
  objects being synchronized and instead will use the default bucket ACL (see
  "gsutil help defacl"). You can override this behavior with the -p option. See
  the `Options section
  <https://cloud.google.com/storage/docs/gsutil/commands/rsync#options>`_ to
  learn how.


<B>LIMITATIONS</B>

  1. The gsutil rsync command will only allow non-negative file modification
     times to be used in its comparisons. This means gsutil rsync will resort to
     using checksums for any file with a timestamp before 1970-01-01 UTC.

  2. The gsutil rsync command considers only the live object version in
     the source and destination buckets when deciding what to copy / delete. If
     versioning is enabled in the destination bucket then gsutil rsync's
     replacing or deleting objects will end up creating versions, but the
     command doesn't try to make any noncurrent versions match in the source
     and destination buckets.

  3. The gsutil rsync command does not support copying special file types
     such as sockets, device files, named pipes, or any other non-standard
     files intended to represent an operating system resource. If you run
     gsutil rsync on a source directory that includes such files (for example,
     copying the root directory on Linux that includes /dev ), you should use
     the -x flag to exclude these files. Otherwise, gsutil rsync may fail or
     hang.

  4. The gsutil rsync command copies changed files in their entirety and does
     not employ the
     `rsync delta-transfer algorithm <https://rsync.samba.org/tech_report/>`_
     to transfer portions of a changed file. This is because Cloud Storage
     objects are immutable and no facility exists to read partial object
     checksums or perform partial replacements.

<B>OPTIONS</B>
  -a predef-acl  Sets the specified predefined ACL on uploaded objects. See
                 "gsutil help acls" for further details. Note that rsync will
                 decide whether or not to perform a copy based only on object
                 size and modification time, not current ACL state. Also see the
                 -p option below.

  -c             Causes the rsync command to compute and compare checksums
                 (instead of comparing mtime) for files if the size of source
                 and destination match. This option increases local disk I/O and
                 run time if either src_url or dst_url are on the local file
                 system.

  -C             If an error occurs, continue to attempt to copy the remaining
                 files. If errors occurred, gsutil's exit status will be
                 non-zero even if this flag is set. This option is implicitly
                 set when running "gsutil -m rsync...".

                 NOTE: -C only applies to the actual copying operation. If an
                 error occurs while iterating over the files in the local
                 directory (e.g., invalid Unicode file name) gsutil will print
                 an error message and abort.

  -d             Delete extra files under dst_url not found under src_url. By
                 default extra files are not deleted.

                 NOTE: this option can delete data quickly if you specify the
                 wrong source/destination combination. See the help section
                 above, "BE CAREFUL WHEN USING -d OPTION!".

  -e             Exclude symlinks. When specified, symbolic links will be
                 ignored. Note that gsutil does not follow directory symlinks,
                 regardless of whether -e is specified.

  -i             Skip copying any files that already exist at the destination,
                 regardless of their modification time.

  -j <ext,...>   Applies gzip transport encoding to any file upload whose
                 extension matches the -j extension list. This is useful when
                 uploading files with compressible content (such as .js, .css,
                 or .html files) because it saves network bandwidth while
                 also leaving the data uncompressed in Google Cloud Storage.

                 When you specify the -j option, files being uploaded are
                 compressed in-memory and on-the-wire only. Both the local
                 files and Cloud Storage objects remain uncompressed. The
                 uploaded objects retain the Content-Type and name of the
                 original files.

                 Note that if you want to use the top-level -m option to
                 parallelize copies along with the -j/-J options, your
                 performance may be bottlenecked by the
                 "max_upload_compression_buffer_size" boto config option,
                 which is set to 2 GiB by default. This compression buffer
                 size can be changed to a higher limit, e.g.:

                   gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \
                     -m rsync -j html,txt /local/source/dir gs://bucket/path

  -J             Applies gzip transport encoding to file uploads. This option
                 works like the -j option described above, but it applies to
                 all uploaded files, regardless of extension.

                 CAUTION: If you use this option and some of the source files
                 don't compress well (e.g., that's often true of binary data),
                 this option may result in longer uploads.

  -n             Causes rsync to run in "dry run" mode, i.e., just outputting
                 what would be copied or deleted without actually doing any
                 copying/deleting.

  -p             Causes ACLs to be preserved when objects are copied. Note that
                 rsync will decide whether or not to perform a copy based only
                 on object size and modification time, not current ACL state.
                 Thus, if the source and destination differ in size or
                 modification time and you run gsutil rsync -p, the file will be
                 copied and ACL preserved. However, if the source and
                 destination don't differ in size or checksum but have different
                 ACLs, running gsutil rsync -p will have no effect.

                 Note that this option has performance and cost implications
                 when using the XML API, as it requires separate HTTP calls for
                 interacting with ACLs. The performance issue can be mitigated
                 to some degree by using gsutil -m rsync to cause parallel
                 synchronization. Also, this option only works if you have OWNER
                 access to all of the objects that are copied.

                 You can avoid the additional performance and cost of using
                 rsync -p if you want all objects in the destination bucket to
                 end up with the same ACL by setting a default object ACL on
                 that bucket instead of using rsync -p. See 'gsutil help
                 defacl'.

  -P             Causes POSIX attributes to be preserved when objects are
                 copied.  With this feature enabled, gsutil rsync will copy
                 fields provided by stat. These are the user ID of the owner,
                 the group ID of the owning group, the mode (permissions) of the
                 file, and the access/modification timestamps of the file. For
                 downloads, these attributes will only be set if the source
                 objects were uploaded with this flag enabled.

                 On Windows, this flag will only set and restore access time and
                 modification time. This is because Windows doesn't have a
                 notion of POSIX uid/gid/mode.

  -R, -r         The -R and -r options are synonymous. Causes directories,
                 buckets, and bucket subdirectories to be synchronized
                 recursively. If you neglect to use this option gsutil will make
                 only the top-level directory in the source and destination URLs
                 match, skipping any sub-directories.

  -u             When a file/object is present in both the source and
                 destination, if mtime is available for both, do not perform
                 the copy if the destination mtime is newer.

  -U             Skip objects with unsupported object types instead of failing.
                 Unsupported object types are Amazon S3 Objects in the GLACIER
                 storage class.

  -x pattern     Causes files/objects matching pattern to be excluded, i.e., any
                 matching files/objects are not copied or deleted. Note that the
                 pattern is a `Python regular expression
                 <https://docs.python.org/3/howto/regex.html>`_, not a wildcard
                 (so, matching any string ending in "abc" would be specified
                 using ".*abc$" rather than "*abc"). Note also that the exclude
                 path is always relative (similar to Unix rsync or tar exclude
                 options). For example, if you run the command:

                   gsutil rsync -x "data.[/\].*\.txt$" dir gs://my-bucket

                 it skips the file dir/data1/a.txt.

                 You can use regex alternation to specify multiple exclusions,
                 for example:

                   gsutil rsync -x ".*\.txt$|.*\.jpg$" dir gs://my-bucket

                 skips all .txt and .jpg files in dir.

                 NOTE: When using the Windows cmd.exe command line interpreter,
                 use ``^`` as an escape character instead of ``\`` and escape
                 the ``|`` character. When using Windows PowerShell, use ``'``
                 instead of ``"`` and surround the ``|`` character with ``"``.

  -y pattern     Similar to the -x option, but the command will first skip
                 directories/prefixes using the provided pattern and then
                 exclude files/objects using the same pattern. This is usually
                 much faster, but won't work as intended with negative
                 lookahead patterns. For example, if you run the command:

                   gsutil rsync -y "^(?!.*\.txt$).*" dir gs://my-bucket

                 This would first exclude all subdirectories unless they end in
                 .txt before excluding all files except those ending in .txt.
                 Running the same command with the -x option would result in all
                 .txt files being included, regardless of whether they appear in
                 subdirectories that end in .txt.

-i   i'  c                     [        5         g)zDCalled when rsync command is killed with SIGINT, SIGQUIT or SIGTERM.N)CleanUpTempFiles)
signal_numcur_stack_frames     'platform/gsutil/gslib/commands/rsync.py_HandleSignalsrP     s        c                     [          H&  n U R                  (       a  M  U R                  5         M(     [          HT  n [        R                  R                  U R                  5      (       d  M3   [        R                  " U R                  5        MV     g! [         a-  n[        R                  " SU R                  U5         SnAM  SnAff = f)ao  Cleans up temp files.

This function allows the main (RunCommand) function to clean up at end of
operation, or if gsutil rsync is interrupted (e.g., via ^C). This is necessary
because tempfile.NamedTemporaryFile doesn't allow the created file to be
re-opened in read mode on Windows, so we have to use tempfile.mkstemp, which
doesn't automatically delete temp files.
z;Failed to close and delete temp file "%s". Got an error:
%sN)
_tmp_filesclosedcloseospathisfilenameunlink	Exceptionloggingdebug)fileobjes     rO   rL   rL   #  s     g>>>mmo  g	ww~~gll##
		',,   JLL!	 	s   + B
C"CCc                 N   UR                   [        R                  :X  d  U R                  (       d  g[	        UR
                  5      nUR                  5       (       aK  [        R                  R                  UR                  5      (       a  U R                  R                  SU5        gg)z5Arg checker that skips symlinks if -e flag specified.TzSkipping symbolic link %s...F)diff_actionr@   REMOVEexclude_symlinksr   src_url_str	IsFileUrlrV   rW   islinkobject_nameloggerinfo)command_instancediff_to_applyexp_src_urls      rO   _DiffToApplyArgCheckerrm   @  sv    :#4#44

+
+$]%>%>?+0G0G!H!H  !?M	rQ   c	                    [        U5      n	[        U5      n
U	R                  5       (       a  U[        :w  d  U
R                  5       (       aG  U[        :  a  U R	                  SU5        [        U	R                  S5       n[        U5      nSSS5        OeU[        :w  d  U
R                  5       (       aF  U[        :  a  U R	                  SU5        [        U	R                  S5       n[        U5      nSSS5        U
R                  5       (       a  U[        :w  aG  U[        :  a  U R	                  SU5        [        U
R                  S5       n[        U5      nSSS5        OPU[        :w  aF  U[        :  a  U R	                  SU5        [        U
R                  S5       n[        U5      nSSS5        X4Xx4$ ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N4= f! , (       d  f       NE= f)a  Computes any file checksums needed by _CompareObjects.

Args:
  logger: logging.logger for outputting log messages.
  src_url_str: Source URL string.
  src_size: Source size
  src_crc32c: Source CRC32c.
  src_md5: Source MD5.
  dst_url_str: Destination URL string.
  dst_size: Destination size
  dst_crc32c: Destination CRC32c.
  dst_md5: Destination MD5.

Returns:
  (src_crc32c, src_md5, dst_crc32c, dst_md5)
zComputing CRC32C for %s...rbNzComputing MD5 for %s...)	r   re   _NArH   ri   openrg   r%   r&   )rh   rd   src_size
src_crc32csrc_md5dst_url_strdst_size
dst_crc32cdst_md5src_urldst_urlfps               rO   _ComputeNeededFileChecksumsr|   M  s   & !-' -'SG--//	G	0+>##T*b:2>
 +*	C7,,..	G	-{;##T*b4R8 +S	G	0+>##T*b:2>
 +*	C	G	-{;##T*b4R8 +
z	33% +*
 +* +*
 +*s0   =F1#GGG$1
F?
G
G!$
G2c                    [        XS9nUu  pEn[        R                  " US[        R                  S9n [        [        XXF5      U5        UR                  5         g! [         ab  nU R                  R                  SU< SU< 35        U R                  R                  [        R                  " 5       5        SU l         SnANxSnAff = f)aO  Worker function for listing files/objects under to be sync'd.

Outputs sorted list to out_file_name, formatted per _BuildTmpOutputLine. We
sort the listed URLs because we don't want to depend on consistent sort
order across file systems and cloud providers.

Args:
  cls: Command instance.
  args_tuple: (base_url_str, out_file_name, desc), where base_url_str is
              top-level URL string to list; out_filename is name of file to
              which sorted output should be written; desc is 'source' or
              'destination'.
  thread_state: gsutil Cloud API instance to use.
thread_statewmodeencodingz-Caught non-retryable exception while listing z:    N)r    iorq   r   UTF8
_BatchSort_FieldedListingIteratorr[   rh   errorr]   	traceback
format_excnon_retryable_listing_failuresrU   )	cls
args_tupler   
gsutil_apibase_url_strout_filenamedescout_filer_   s	            rO   _ListUrlRootFuncr   {  s     #3B*'1$<t WW\innE(+&sK .. 
 + JJ"A' ( JJY))+,)*C&&+s   A 
C%ACCc              #   (  #    [         R                  " U R                  5       Hj  n[         R                  R	                  U R                  U5      n[         R                  R                  U5      (       d  MS  [        [        U5      S5      v   Ml     g7f)a  A generator that yields a BLR for each file in a local directory.

   We use this function instead of WildcardIterator for listing a local
   directory without recursion, because the glob.globi implementation called
   by WildcardIterator skips "dot" files (which we don't want to do when
   synchronizing to or from a local directory).

Args:
  base_url: URL for the directory over which to iterate.

Yields:
  BucketListingObject for each file in the directory.
N)rV   listdirrg   rW   joinrX   r
   r   )base_urlfilenames     rO   _LocalDirIteratorr     s`      **X112hww||H00(;H	ww~~h 4X >EE 3s   A0B6Bc           	   #     #    [        U5      nUR                  S:X  a  U R                  (       d  [        U5      nOU R                  (       a  SUR	                  S5      -  nOSUR	                  S5      -  nSSSSS	S
[
        -  /nU R                  (       a1  UR                  S
[        -  S
[        -  S
[        -  S
[        -  /5        U R                  b  X@R                  U R                  4OSn[        UUU R                  UU R                   U R"                  S9R%                  US9nSn	U GH+  n
U
R&                  n[)        XS9(       a  M   U R                   (       aE  UR+                  5       (       a0  [,        R.                  R1                  UR2                  5      (       a  Mv  U R                  (       ai  UR4                  [7        UR4                  5      S nUR9                  UR:                  5      (       a  USS nU R                  R=                  U5      (       a  M  U	S-  n	U	[>        -  S:X  a  U R"                  RA                  SX95        [C        U
5      v   GM.     g7f)aF  Iterator over base_url_str formatting output per _BuildTmpOutputLine.

Args:
  cls: Command instance.
  gsutil_api: gsutil Cloud API instance to use for bucket listing.
  base_url_str: The top-level URL string over which to iterate.
  desc: 'source' or 'destination'.

Yields:
  Output line formatted per _BuildTmpOutputLine.
filez%s/**/\z%s/*crc32cmd5HashrY   sizetimeCreatedzmetadata/%sN)
project_idexclude_tupleignore_symlinksrh   )bucket_listing_fieldsr   )blrr   zAt %s listing %d...)"r   schemerecursion_requestedr   rstripr3   preserve_posix_attrsextendr,   r2   r0   r;   exclude_patternexclude_dirsrI   r   rc   rh   IterObjectsstorage_urlr   re   rV   rW   rf   rg   
url_stringlen
startswithdelimmatch_PROGRESS_REPORT_LISTING_COUNTri   _BuildTmpOutputLine)r   r   r   r   r   iteratorwildcardfieldsr   ir   urlstr_to_checks                rO   r   r     s     ",/(__s'>'> *H
<..u55h,--e44h
"F mm
*
$
)
#
(
"
(
"	 	 !$ 3 3 ? 	""EI  &>>#,,zz ';"( ' *  !c //C- 
s''
 ^^C(;(;$<$=>l		 	 	+	+#AB'				"	"<	0	0FA))Q.	jjoo+T5
c
""9 s   IIc           
         [         n[        n[        n[        n[        n[         n[         n[        nU R                  n	U	R                  5       (       ac  [        R                  " U	R                  5      u
  n    pp;pn
[        U5      n[        U5      n[        U5      nUS:  a  [         nUS:  a  [         nGOU	R                  5       (       Ga  U R                  R                  nU R                  R                  b  [        U R                  [         [         5      u  p [        U5      nU(       a   U[         ::  a  [#        SU	R$                  5        U[        [&        R&                  " 5       5      [(        -   :  a  [+        SU	R$                  5        [1        U R                  U	R$                  5      nUR2                  R4                  nUR6                  nUR8                  nUR:                  n[=        U R                  R>                  5      nU R                  R@                  =(       d    [        nU R                  RB                  =(       d    [        nO[E        SU	RF                  -  5      e[I        U	R$                  5      UUUUUUUUU/
nU Vs/ s H"  n[J        RL                  " [O        U5      5      PM$     nnSRQ                  U5      S-   $ ! [,         a     [/        SU	R$                  5        [         n GNcf = fs  snf )a  Builds line to output to temp file for given BucketListingRef.

Args:
  blr: The BucketListingRef.

Returns:
  The output line, formatted as
  _EncodeUrl(URL)<sp>size<sp>time_created<sp>atime<sp>mtime<sp>mode<sp>uid<sp>
  gid<sp>crc32c<sp>md5 where md5 will only be present for cloud URLs that
  aren't composite objects. A missing field is populated with '-', or -1 in
  the case of atime/mtime/time_created.
r   mtimezGot unexpected URL type (%s) 
))r6   rp   r4   r5   r   re   rV   statrg   longr.   
IsCloudUrlroot_objectr   metadatar*   r3   r?   r   timerG   r=   
ValueErrorr>   r/   r   permissionsatimeuidgidr-   r   r   r   r   r   
_EncodeUrlsixensure_textstrr   )r   r   r   r   md5r   r   time_createdr   r   _r   found_m	mtime_strposix_attrsattrsr   s                    rO   r   r     sQ    %&##	$
%,##]]__57WWS__5M2D!Q5KEKEd#Dqyeqye
~~??D
+;
//:w0g Yu'

84		$66
gs~~
6 @
//3>>+k))deOOcOOc *#//*E*EFL__##*sF
//
!
!
(SC
9CJJF
GG 

			% -2
2Eq3??3q6"E%
2	%4	?   	#..1	< 3s   A1K )K2&K/.K/c                 j    [         R                  " U 5      n [        R                  R	                  U SS9$ )zEncodes url_str with quote plus encoding and UTF8 character encoding.

We use this for all URL encodings.

Args:
  url_string (unicode): String URL to encode.

Returns:
  (str) A string encoded using urllib's `quote_plus()` method.
   ~)safe)r   
ensure_strr   parse
quote_plus)r   s    rO   r   r   V  s,     ~~j)*		 	 $	 	77rQ   c                     [         R                  R                  U 5      n[        R                  (       a  UR                  [        R                  5      nU$ )zInverts encoding from `_EncodeUrl()`.

Args:
  enc_url_string (str): String containing UTF-8-decodable characters that were
      encoded using urllib's `quote_plus()`.

Returns:
  (unicode) A decoded URL.
)r   r   unquote_plusr   PY2decoder   r   )enc_url_stringr   s     rO   
_DecodeUrlr   i  s6     	!!.1#WW
**Y^^
$C	*rQ   c           	         [         R                  " SSS5      n/ n  [        [        X5      5      nU(       d  O[        R
                  " SUR                  [        U5      4-  S[        R                  S9nUR                  U5        UR                  [        R                  " SR                  U5      5      5        UR                  5         UR!                  S5        M  UR#                  [$        R&                  " U6 5         U H4  n UR5                  5         [6        R8                  " UR                  5        M6     g! [(         aR  nUR*                  [*        R,                  :X  a.  [/        S	R                  [0        R2                  " S
5      5      5      ee SnAff = f! [:         a-  n[<        R>                  " SUR                  U5         SnAM  SnAff = f! U Hn  n UR5                  5         [6        R8                  " UR                  5        M6  ! [:         a-  n[<        R>                  " SUR                  U5         SnAMh  SnAff = f   f = f)a  Sorts input lines from in_iter and outputs to out_file.

Sorts in batches as input arrives, so input file does not need to be loaded
into memory all at once. Derived from Python Recipe 466302: Sorting big
files the Python 2.4 way by Nicolas Lehuen.

Sorted format is per _BuildTmpOutputLine. We're sorting on the entire line
when we could just sort on the first record (URL); but the sort order is
identical either way.

Args:
  in_iter: Input iterator.
  out_file: Output file.
GSUtilrsync_buffer_linesi }  z%s-%06izw+r    r   r   zSynchronization failed because too many open file handles were needed while building synchronization state. Please see the comments about rsync_buffer_lines in your .boto config file for a possible way to address this problem.Nz8Failed to remove rsync chunk file "%s". Got an error:
%s) r	   getintsortedr   r   rq   rY   r   r   r   appendwriter   	text_typer   flushseek
writelinesheapqmergeIOErrorerrnoEMFILEr   textwrapwraprU   rV   remover[   r\   r]   )in_iterr   buffer_sizechunk_filescurrent_chunkoutput_chunkr_   
chunk_files           rO   r   r   z  s   ( h(<eD++ 
VG9:mWWY(--[9I)JJ"&&/nn6l &rww}'=>?  [12 "
 
		*//" " 
 
ww%,,TYY
--678 9 9 

   GOOQ	  	  	 "
 
		*//"  GOOQ	  	  	 "sg   C#D; 0F;
FAFFG 
G$"GGI
0HI

I	"H?	9I
?I	I
c                   <    \ rS rSrSrS rS rS rS rS r	S r
S	rg
)_DiffIteratori  z7Iterator yielding sequence of RsyncDiffToApply objects.c           
      f   Xl         UR                  U l        UR                  U l        UR                  U l        U R                   R                  U l        X l        X0l        UR                  U l        UR                  U l	        UR                  U l
        U R                  R                  S5        [        R                  " SSS9n[        R                  " SSS9nUR                  U l        UR                  U l        ["        R%                  U5        ["        R%                  U5        UR'                  5         UR'                  5         [)        U R
                  R*                  U R                  S4U R                  R*                  U R                   S4/5      nSUl        S	/nUR/                  [0        U[2        U[4        UR6                  R8                  S
S9  UR,                  (       a  [;        S5      e[=        U R                  S5      U l        [=        U R                   S5      U l         ["        R%                  U R>                  5        ["        R%                  U R@                  5        URC                  5       (       at  URE                  5       (       a_  U R                  (       aN  [G        [)        U R>                  5      5      U l$        U RK                  5         U R>                  RM                  S5        [G        [)        U R>                  5      5      U l$        [G        [)        U R@                  5      5      U l'        g )Nz!Building synchronization state...zgsutil-rsync-src-F)prefixdeletezgsutil-rsync-dst-sourcedestinationr   r   T)arg_checkerparallel_operations_overridefail_on_errorz/Caught non-retryable exception - aborting rsyncr)(command_objcompute_file_checksumsdelete_extrasr   rh   base_src_urlbase_dst_urlr   preserve_posixskip_old_filesignore_existingri   tempfileNamedTemporaryFilerY   sorted_list_src_file_namesorted_list_dst_file_namerS   r   rU   iterr   r   Applyr   _RootListingExceptionHandlerr   ParallelOverrideReasonSPEEDr   rq   sorted_list_src_filesorted_list_dst_filer   re   r   sorted_src_urls_it_ValidateObjectAccessr   sorted_dst_urls_it)selfr  r  r  temp_src_filetemp_dst_file	args_itershared_attrss           rO   __init___DiffIterator.__init__  s   ""-"D"DD$22D*>>D""))DK$$%::D%44D&66DKK89
 //7J7<>M//7J7<>M%2%7%7D"%2%7%7D"m$m$ 
 ((**	
 ((**	
 I 23K.45L$#%0%G%G%M%M   11NOO
 !%T%C%CS ID $T%C%CS IDd//0d//0!!l&<&<&>&> :
t((
)!+d
  "
$$Q' 9T&&')D8T&&')DrQ   c                 D   [         R                  " 5       nU R                   HE  nU R                  U5      u
  n      pEpg  n[	        UUUUS9u  pU(       a  M4  UR                  U	5        MG     U(       a/  U H  n	U R                  R                  U	5        M      [        S5      eg)a   Validates that the user won't lose access to the files if copied.

Iterates over the src file list to check if access will be maintained. If at
any point we would orphan a file, a list of errors is compiled and logged
with an exception raised to the user.
)r   r   r   zPThis sync will orphan file(s), please fix their permissions before trying again.N)	collectionsdequer  _ParseTmpFileLiner<   r   rh   criticalr   )
r  errorsry   rd   r   src_modesrc_uidsrc_gidvaliderrs
             rO   r  #_DiffIterator._ValidateObjectAccess
  s      F**

 
 
) @k1aA1a/4;4;5=?je Uc + #S!  @ A A rQ   c                     UR                  SS5      u
  p#pEpgpn
n[        U5      [        U5      [        U5      [        U5      [        U5      [        U5      [        U5      [        U	5      U
UR	                  5       4
$ )ae  Parses output from _BuildTmpOutputLine.

Parses into tuple:
  (URL, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5)
where crc32c and/or md5 can be _NA and atime/mtime/time_created can be
NA_TIME.

Args:
  line: The line to parse.

Returns:
  Parsed tuple: (url, size, time_created, atime, mtime, mode, uid, gid,
                 crc32c, md5)
N	   )rsplitr   intr   strip)r  lineencoded_urlr   r   r   r   r   r   r   r   r   s               rO   r(  _DiffIterator._ParseTmpFileLine!  st      KKa 
[U#F;D	\UUD	CC		 rQ   c                     [        U5      R                  5       (       a1  U[        :X  a'  U[        :X  a  U R                  R	                  SU5        gg)zWarns if given url_str is a cloud URL and is missing both crc32c and md5.

Args:
  url_str: Destination URL string.
  crc32c: Destination CRC32c.
  md5: Destination MD5.

Returns:
  True if issued warning.
zKFound no hashes to validate %s. Integrity cannot be assured without hashes.TF)r   r   rp   rh   warn)r  url_strr   r   s       rO   _WarnIfMissingCloudHash%_DiffIterator._WarnIfMissingCloudHash?  sG     	W%0022v}s

kk rQ   c                 
   U[         :  nU[         :  nU R                  =(       d9    [        U5      R                  5       =(       a    [        U5      R                  5       nU R                  (       a  SX4$ U R
                  (       a  U(       a  U(       a	  X8:  a  SX4$ U(       d  U(       a  U(       a  X8:g  =(       d    X':g  X4$ X':w  a  SX4$ [        U R                  UUUUUUU	U
5	      u  pEpU[        :w  a,  U
[        :w  a"  U R                  R                  SX5        XZ:g  X4$ U[        :w  a-  U	[        :w  a#  U R                  R                  SUU5        XI:g  X4$ U R                  XU5      (       d  U R                  XiU
5        SX4$ )a)  Returns whether src should replace dst object, and if mtime is present.

Uses mtime, size, or whatever checksums are available.

Args:
  src_url_str: Source URL string.
  src_size: Source size.
  src_mtime: Source modification time.
  src_crc32c: Source CRC32c.
  src_md5: Source MD5.
  dst_url_str: Destination URL string.
  dst_size: Destination size.
  dst_mtime: Destination modification time.
  dst_crc32c: Destination CRC32c.
  dst_md5: Destination MD5.

Returns:
  A 3-tuple indicating if src should replace dst, and if src and dst have
  mtime.
FTzComparing md5 for %s and %szComparing crc32c for %s and %s)r6   r	  r   r   r  r  r|   rh   rp   r]   r<  )r  rd   rr   	src_mtimers   rt   ru   rv   	dst_mtimerw   rx   has_src_mtimehas_dst_mtime
use_hashess                 rO   _CompareObjects_DiffIterator._CompareObjectsT  s   T 'M'M-- C'4??A B'4??A  M00-M00-M$ #"MB B=///J
0,J #~'S.
kk5{P==SZ3.
kk8+#%%}CC''II
"";GD -..rQ   c              #   l  #    [        U R                  R                  R                  S5      5      n[        U R                  R                  R                  S5      5      nSnS=pE Uc  U R
                  R                  5       (       a  SnOU R                  [        U R
                  5      5      u
  pFpxn	ppn[        UU	UUU
S9n[        XAS R                  SS5      5      n[        R                  " U R                  [        U5      SSSU R                  SU R                  S9R                  nUck  U R                   R                  5       (       dL  U R                  [        U R                   5      5      u
  nnnnnnnnnn[        XRS R                  SS5      5      nU(       a  GO{Ub  WW:  a#  [#        UWW[$        R&                  W5      v   SnGOLUU:  a<  U R(                  (       a'  [#        SU[        5       [$        R*                  S5      v   SnGO
[        U5      R-                  5       (       a*  [        U5      R/                  5       (       a  W	[0        :X  a  Wn	U R3                  UWW	WWUWWWW5
      u  nnnU(       a  [#        XEW[$        R&                  U5      v   OzU R4                  (       a=  [7        WWU	UWWWWW
W5
      u  nnU(       a  [#        XEU[$        R8                  U5      v   O,U(       a%  U(       d  [#        XEW[$        R:                  U5      v   SnSnGM  U R(                  (       d  gU(       a'  [#        SU[        5       [$        R*                  S5      v   U R                    HF  n U R                  U 5      u
  n                n[#        SU[        5       [$        R*                  S5      v   MH     g7f)	zfIterates over src/dst URLs and produces a RsyncDiffToApply sequence.

Yields:
  The RsyncDiffToApply.
r   FNT)r   r   r   r   r   \/)ry   rl   src_url_names_containerhave_multiple_srcshas_multiple_top_level_srcsexp_dst_urlhave_existing_dest_subdirr   )r   r  r   r   r  r  IsEmptyr(  nextr9   r   replacer   ConstructDstUrlr   r   r  rA   r@   COPYr
  rb   r   re   r6   rD  r  r7   POSIX_SRC_TO_DSTMTIME_SRC_TO_DST)!r  base_src_url_lenbase_dst_url_lenout_of_src_itemsrd   ru   rr   src_time_created	src_atimer?  r+  r,  r-  rs   rt   r   src_url_str_to_checkdst_url_str_would_copy_torv   r   	dst_atimer@  dst_modedst_uiddst_gidrw   rx   dst_url_str_to_checkshould_replacerA  rB  needs_updater6  s!                                    rO   __iter___DiffIterator.__iter__  s     4,,77>>uEF4,,77>>uEF $$K 		""**,,!
 --d43J3J.KL;"2yg'i.7,3,3-5	7+ ",+,44T3?"A
&1&A&A''.{;&*!%*/++(-"&":":'< =GJ $ 
	&&..00 ,,T$2I2I-JK;!Y	8WJ ",+,44T3?"A
 
 
!58L!L{,E*JOOXG 	G"66  {O4E!+!2!2D: : !-88:: -7799 &)8<8L8L9j';iW964}  ;!+(< <  &?Iy'7w('4
#+| ";[#-#>#>J J= !;!+!<!<hH H
 i l  T;0A'..6 6''151G1G1M.{Aq!Q1aAT;0A'..6 6 (s   N2N4)r  r  r  r	  r
  r  rh   r  r   r  r  r  r  r  r  r  N)__name__
__module____qualname____firstlineno____doc__r#  r  r(  r<  rD  rc  __static_attributes__ rQ   rO   r   r     s)    ?T)lA.<*O/br6rQ   r   c                   $    \ rS rSrSrS rS rSrg)_SeekAheadDiffIteratori  zGWraps _AvoidChecksumAndListingDiffIterator and yields SeekAheadResults.c                     Xl         g Ncloned_diff_iterator)r  rq  s     rO   r#  _SeekAheadDiffIterator.__init__  s     4rQ   c              #      #    U R                    Ha  nUR                  =(       d    SnUR                  [        R                  :X  d  UR                  [        R
                  :X  a  Sn[        US9v   Mc     g 7f)Nr   )
data_bytes)rq  	copy_sizera   r@   rT  rS  r   )r  rk   bytes_to_copys      rO   rc  _SeekAheadDiffIterator.__iter__   s_     22#--2m

#
#z'B'B
B

#
#z'B'B
B
 }55 3s   A3A5rp  N)re  rf  rg  rh  ri  r#  rc  rj  rk  rQ   rO   rm  rm    s    O56rQ   rm  c                       \ rS rSrSrS rSrg)$_AvoidChecksumAndListingDiffIteratori.  zIterator initialized from an existing _DiffIterator.

This iterator yields RsyncDiffToApply objects used to estimate the total work
that will be performed by the DiffIterator, while avoiding expensive
computation.
c                    SU l         UR                  U l        UR                  U l        SU l        [        R
                  " S5      U l        UR                  U l        UR                  U l        UR                  U l	        UR                  U l
        [        UR                  S5      U l        [        UR                  S5      U l        [         R#                  U R                  5        [         R#                  U R                  5        [%        ['        U R                  5      5      U l        [%        ['        U R                  5      5      U l        g )NFdummyr  )r	  r
  r   r  r\   	getLoggerrh   r  r  r  r  rq   r  r  r  r  rS   r   r   r  r  r  )r  initialized_diff_iterators     rO   r#  -_AvoidChecksumAndListingDiffIterator.__init__7  s    #(D2@@D8FFD  D##G,DK1>>D1>>D3BBD4DDD
 !%!;;S!BD $!;;S!BDd//0d//0 9T&&')D8T&&')DrQ   )r  r  r	  r
  r  rh   r  r   r  r  r  r  r  N)re  rf  rg  rh  ri  r#  rj  rk  rQ   rO   ry  ry  .  s    ")rQ   ry  c                    [        XS9nUR                  n[        U5      nUR                  nUR                  [
        R                  :X  a  U R                  (       a  U R                  R                  SU5        gU R                  R                  SU5        UR                  5       (       a"   [        R                  " UR                  5        g UR!                  UR"                  UR                  UR$                  UR&                  S9  gUR                  [
        R*                  :X  Ga  UR,                  n[        U5      nU R                  (       aS  UR                  5       (       a!   [/        UR                  S5       n	 SSS5        U R                  R                  S	X5        g SnUR3                  5       (       a  [5        UUR$                  5      nUR7                  UR"                  UR                  UUR&                  U R8                  S
9n[;        U5      (       a  U R                  R                  SU-  5        O[<        R>                  " 5       nUR@                  [C        [D        RD                  " 5       5      [F        -   :  a  [I        SURJ                  5        URM                  5       (       d  URO                  5       (       a?  URO                  5       (       a  SOSnU R                  RQ                  SXRJ                  5        URR                  (       a  URR                  nO[<        R>                  RU                  / S9n[W        UUU RX                  S9  [<        R>                  " 5       nXl)        [[        XSS9  [\        R^                  " U R                  UUUU [`        UU Rb                  SU Rd                  U Rf                  U RX                  S9nUb2  Uu  n	n  n	U Rh                     U =Rj                  U-  sl5        SSS5        ggUR                  [
        Rp                  :X  Ga<  [        UR                  5      nU R                  (       a  U R                  R                  SU5        gU R                  R                  SURJ                  5        UR@                  n[<        R>                  " 5       n[s        [t        U05      Ul)        UR3                  5       (       ag  [        UR                  5      n[5        XUR$                  5      n URw                  UR"                  UR                  UUR&                  UR$                  S9  g[        UR                  UU RX                  S9  gUR                  [
        R                  :X  GaO  [        UR                  5      nU R                  (       a  U R                  R                  SU5        gU R                  R                  SURJ                  5        [<        R>                  " 5       n[<        R>                  RU                  / S9Ul)        [W        UURR                  SS9  UR3                  5       (       a  [5        XUR$                  5      nUR7                  UR"                  UR                  UUR&                  S/S
9n URw                  UR"                  UR                  UUR&                  UR$                  S9  gg[        SUR                  -  5      e! [         a    U R                  R                  SU5         gf = f! [(         a     gf = f! , (       d  f       GN= f! [0         a.  n
U R                  R                  SUR                  -  5        e Sn
A
ff = f! , (       d  f       g= f! [l         a0  n
U R                  R                  SXRn                  5         Sn
A
gSn
A
ff = f! [x         a  nU R                  R                  SU5        U R                  R                  SURJ                  5        [{        U [}        UR,                  UR                  U[
        R*                  UR~                  5      US9   SnAgSnAff = f! [x         a  nU R                  R                  SU5        U R                  R                  SURJ                  5        [{        U [}        UR,                  UR                  U[
        R*                  UR~                  5      US9   SnAgSnAff = f)zEWorker function for performing the actual copy and remove operations.r~   zWould remove %szRemoving %sz%s was already removed)
generationproviderro   NzCould not open %szWould copy %s to %s)r  r  r   a  %s has a compressed content-encoding, so it will be decompressed upon download; future executions of gsutil rsync with this source object will always download it. If you wish to synchronize such an object efficiently, compress the source objects in place before synchronizing them, rather than (for example) using gsutil cp -Z to compress them on-the-fly (which results in compressed content-encoding).r   StreamszNamed pipeszlWARNING: %s are not supported by gsutil rsync and will likely fail. Use the -x option to exclude %s by name.)additionalProperties)r  T)override)src_obj_metadataheadersis_rsyncgzip_encoded	gzip_extsr  z0Skipping item %s with unsupported object type %szWould set mtime for %sz$Copying mtime from src to dst for %s)r  r  zError while trying to patch: %szkCopying whole file/object for %s instead of patching because you don't have patch permission on the object.z!Would set POSIX attributes for %sz/Copying POSIX attributes from src to dst for %saclzGot unexpected DiffAction (%d))Cr    ru   r   src_posix_attrsra   r@   rb   dryrunrh   ri   re   rV   rZ   rg   FileNotFoundErrorr]   DeleteObjectbucket_namer  r   r   rR  rd   rq   r[   r   r   GetObjectMetadatasource_metadata_fieldsr+   apitools_messagesObjectr   r   r   rG   r=   r   IsFifoIsStreamr:  r   MetadataValuer:   r   rE   r   PerformCopy_RsyncExceptionHandlerr  r  r  
stats_locktotal_bytes_transferredr$   unsupported_typerT  r)   r3   PatchObjectMetadatar   
_RsyncFuncrA   ru  r8   rS  r   )r   rk   r   r   ru   rz   r   rd   ry   r   r_   r  src_generation	type_textcustom_metadatatmp_obj_metadatacopy_resultbytes_transferredr   obj_metadatadst_generationr/  dst_obj_metadatas                          rO   r  r  ^  s   "3B*))+ -'--+*"3"33
zz	jjoo'1	jjoomW-						
))G''
(	

!
!'"5"5")"5"5-4-?-?+2>> " ;   JOO3++K";/G
zz						G''.! /
 
jjoo+W>?55g6=6H6HJ.'99!!!!'~~// : 1
 !!122JJOOM  /557
diik!2_!DD););<^^!1!1!3!3%,%5%5%7%7	]IJJOOM--/ $$,55/-44BB#% C '//33	5 -335$3!+M!--JJ"-KK))mm335 ")4
&1A~~''+<<' ~ #   J$?$?? #=#<#<=G
zz	jjoo.8	jjoo<((*e&--/l2J3FGl					&}'@'@A3G=O=OP	0

(
()<)<)0)<)<)529..4;4F4F	 ) H$ 	#7#6#6#/252J2J	L   J$?$?? #=#<#<=G
zz	jjoo97C	jjooG((*&--/l/66DD! E #l-k.:.C.C=AC 
				3G=O=OP%77%^^7 8 	0

(
()<)<)0)<)<)529..4;4F4F	 ) H 
8 ;(445 6 6c ! 	 **

3W
=
		 ! 	 	 /. 	
**//-0C0CC
D
	@ ~' 5

J!3!3	5 	554   	0
**

<c
B
**// ++- S%m&?&?&3&?&?&0oo}7N7NP #/	0	0^   	0
**

<c
B
**// ++- S%m&?&?&3&?&?&0oo}7N7NP #/	0	0s   + [" :\ %\0 ;\=\0 #H>]< !]+7]<  ;^9 ;a "&\
\
\\
\-(\0 -\0 0
](:)]##](+
]95]< 9]< <
^6&^11^69
aBaa
c0 Bc++c0c                 L    U R                   R                  [        U5      5        g)zDSimple exception handler for exceptions during listing URLs to sync.N)rh   r   r   r   r_   s     rO   r  r  !  s    **3q6rQ   c                     U R                   R                  [        U5      5        U =R                  S-  sl        U R                   R	                  S[
        R                  " 5       5        g)z9Simple exception handler to allow post-completion status.r   z*

Encountered exception while syncing:
%s
N)rh   r   r   op_failure_countr]   r   r   r  s     rO   r  r  &  sJ    **3q6!**C'')+rQ   c                     ^  \ rS rSrSr\R                  " S/ \SSSSSS\R                  \R                  /\R                  \R                  " S5      /S	9r\R                  " SS
S/SS\0 S9rU 4S jrS rS rS rSrU =r$ )RsyncCommandi.  z'Implementation of gsutil rsync command.rsync   za:cCdenpPriRuUx:y:j:JTFr   )command_name_aliasesusage_synopsismin_argsmax_argssupported_sub_argsfile_url_okprovider_url_okurls_start_arggs_api_supportgs_default_apiargparse_argumentssyncsynchronizecommand_helpz.Synchronize content of two buckets/directories)	help_namehelp_name_aliases	help_typehelp_one_line_summary	help_textsubcommand_help_textc                 d  > [        U R                  5        SS/nU R                   VVs/ s H  u  p#UPM	     nnnSU;  a!  US/-  nU R                  R                  S5        SU;   ak  U R	                  SS9u  nu  pV[        U5      R                  5       (       a9  [        U5      R                  5       (       a  U R                  R                  S	5        [        U0 S
[        S5      _S[        S5      _S[        S5      _S[        S5      _S[        S5      _S[        S5      _S[        S5      _S[        S5      _S[        S5      _S[        S5      _S[        S5      _S[        S5      _S [        S5      _S![        S"5      _S#[        S$5      _S%[        S&5      _S'9n[        TU ]-  U5      $ s  snnf )(Nstorager  -ez--no-ignore-symlinkszBy default, gsutil copies file symlinks, but, by default, this command (run via the gcloud storage shim) does not copy any symlinks.-PF)should_update_sub_opts_and_argszFor preserving POSIX with rsync downloads, gsutil aborts if a single download will result in invalid destination POSIX. However, this command (run via the gcloud storage shim) will skip invalid copies and still perform valid copies.-az--predefined-acl-cz--checksums-only-Cz--continue-on-error-dz&--delete-unmatched-destination-objectsz--ignore-symlinks-iz--no-clobber-Jz--gzip-in-flight-all-jz--gzip-in-flight-nz	--dry-runz--preserve-posix-pz--preserve-acl-Rz--recursive-r-Uz--skip-unsupported-uz--skip-if-dest-has-newer-mtime-xz	--exclude)gcloud_commandflag_map)r   sub_optsrh   r:  ParseSubOptsr   r   re   rC   rB   superget_gcloud_storage_args)	r  r  flagr   	flag_keyssource_pathdestination_pathgcloud_storage_map	__class__s	           rO   r  $RsyncCommand.get_gcloud_storage_argsI  s!   +DMM:)N%)]]3]'$]I39/00n
kk y+/+<+<*/ ,= ,1(a	(+
{
+
6
6
8
8
/
0
:
:
<
<C	D *%
#$67
#$67
 #$9:
 #$LM	

 #$78
 #N3
 #$:;
 #$67
 #K0
 #$67
 #$45
 #M2
 #M2
 #$89
 #$DE
  #K0!
* 7*+=>>O 4s   F,c                     [         R                  " UU R                  U R                  UU R                  S9u  p4U(       d  [        SU-  5      eU$ )aS  Sanity checks that URL names an existing container.

Args:
  url_str: URL string to check.
  treat_nonexistent_object_as_subdir: indicates if should treat a
                                      non-existent object as a subdir.

Returns:
  URL for checked string.

Raises:
  CommandException if url_str doesn't name an existing container.
)rh   zarg (%s) does not name a directory, bucket, or bucket subdir.
If there is an object with the same path, please add a trailing
slash to specify the directory.)r   ExpandUrlToSingleBlrr   r   rh   r   )r  r;  "treat_nonexistent_object_as_subdirr   have_existing_containers        rO   _InsistContainerRsyncCommand._InsistContainerv  sX     &1%E%E*{{&"S #,.567 7 JrQ   c           
         U R                  5         SU l        [        R                  " 5       U l        [        5       (       dP  U R                  (       a   U R                  R                  [        5        OU R                  R                  [        5        U R                  U R                  S   S5      nU R                  U R                  S   S5      nUR                  5       =(       a0    UR                  5       =(       a    UR                  UR                  :g  n[        UR!                  5       UR                  5       UR!                  5       UR                  5       UU R"                  UR                  UR                  /S9  [%        UR                  5       U R&                  U R(                  SU R*                  S9U l        SU l        Sn[1        5        H  n[3        U[4        5        M     U R7                  SSU R8                  R:                  SS	9u  pg[<        R>                  " UU R@                  Xg-  S
9  [C        XU5      n[E        [G        U5      5      n	U R                  RI                  S5        [J        RJ                  " 5       n
 U RM                  [N        U[P        U[R        SU	S9  [U        5         [J        RJ                  " 5       nX-
  U l+        [Y        U R                  U RV                  5      U l-        [        U RZ                  U RV                  U R                  S9  U R.                  (       a/  U R.                  (       a  SOSn[]        SU R.                  X4-  5      eg! [U        5         f = f)z*Command entry point for the rsync command.r   Fr   T)has_file_srchas_cloud_srchas_file_dsthas_cloud_dstis_daisy_chainuses_fanprovider_types)r  r  )r  r  N)process_countthread_countr  print_macos_warning)worker_countzStarting synchronization...)r  r  seek_ahead_iterator)avg_throughputtotal_elapsed_timer  sr   z/%d file%s/object%s could not be copied/removed.)/
_ParseOptsr  r   
CreateLockr  r   r	  rh   r:  r(   r'   r  argsr   r   r   re   parallel_operationsr"   skip_unsupported_objectspreserve_aclr   r  r  r   r   rP   _GetProcessAndThreadCountr  r  r   .TriggerReauthForDestinationProviderIfNecessaryr   r   rm  ry  ri   r   r  r  r  rm   rL   r  rF   total_bytes_per_secondr   )r  ry   rz   r  r"  rM   r  r  diff_iteratorr  
start_timeend_time
plural_strs                rO   
RunCommandRsyncCommand.RunCommand  s   OO#$D  1;;=DO!!		$	$,-23##DIIaL%8G##DIIaL$7G((* 7w/A/A/C 7nn6 W->->-@.5.@.@.B-4->->-@.5.@.@.B/=)-)A)A07/OQ #?%%00#2D D CL&(
J7 ) #'"@"@%)%@%@%F%F!	 #A ##M
 >>"1 "$9M
 1,];= 	KK23J	
jj'3#%8  : yy{H&3D"5$$d&=&=#?D2222 $ < <>
 //3RjN"33ZLM N N  s    !M M$c                    SU l         SU l        SU l        SU l        SU l        SU l        SU l        SU l        S U l        SU l	        SU l
        SU l        S nS U l        SnS nS nU R                  (       Ga}  U R                   GHl  u  pVUS:X  a	  UnSU l        US:X  a	  SU l        M$  US:X  a	  SU l        M3  US:X  a	  SU l        MB  US:X  a	  SU l         MQ  US:X  a2  SnUR                  S	5       Vs/ s H  owR                  5       PM     nnM  US
:X  a
  Sn[         nM  US:X  a	  SU l        M  US:X  a	  SU l        M  US:X  a   SU l        ["        (       d  [%        5         M  M  US:X  d  US:X  a	  SU l        M  US:X  a
  SU l	        GM  US:X  a
  SU l
        GM  US:X  a
  SU l        GM"  US:X  d	  US:X  d  GM1  US:X  a  SU l        U(       d  [)        S5      e [*        R,                  " U5      U l        GMo     U R                  (       a  U(       a  [)        S5      eU(       a  U(       a  [)        S5      eX l        U=(       d    UU l        [5        UU R                  U R                  S9$ s  snf ! [*        R.                   a    [)        SU-  5      ef = f)NFr  Tr  r  r  r  r  ,r  r  r  r  r  r  r  r  r  r  z-yzInvalid blank exclude filterzInvalid exclude filter (%s)z:Specifying both the -p and -a options together is invalid.z:Specifying both the -j and -J options together is invalid.)
canned_aclr  r  )rc   continue_on_errorr
  r  r   r	  r  r   r   r  r  r  cannedr  splitr5  r#   rD   r1   r   r   recompiler   r  r  r!   )r  r
  r  gzip_arg_extsgzip_arg_alloaxs           rO   r  RsyncCommand._ParseOpts  s    "D #DDD %D"'DDKDDD D$)D! J DK
 LML}}}--$!9*$+9(,$
% $Y#'$
 $Y#$
$Y"&$
$Y,./ggcl;l779l-;-$Y,',$Y$+$Y"$
$Y&*$
#') $Y!t)%)$
"$Y $$
$Y!%$
$Y*.$
'$Y!t)$Y $D"#ABBF#%::a=D W  ^ Z
FH H
FH H$"2lDN&&!%!>!>@ @O <6  F"#@1#DEEFs   .I$I))#J)r  r	  r  r
  r  r   r   rc   r  r  r  r  r  r   r   r  r  r  r  r  r  r  )re  rf  rg  rh  ri  r   CreateCommandSpec	_SYNOPSISr   XMLJSONr   MakeNCloudOrFileURLsArgumentcommand_specHelpSpec_DETAILED_HELP_TEXT	help_specr  r  r  r  rj  __classcell__)r  s   @rO   r  r  .  s    / **0!oo{'7'78 %%)FFqIJL, /L#)+?Z6ZNxZ@ Z@rQ   r  ro  )ri  
__future__r   r   r   r   r&  r   r   r   	itertoolsr   r\   rV   r  r  r   r   r   sysr   	six.movesr   botor	   crcmodgslib.bucket_listing_refr
   gslib.cloud_apir   r   gslib.commandr   r   gslib.commands.cpr   gslib.command_argumentr   gslib.cs_api_mapr   gslib.exceptionr   gslib.metricsr   "gslib.plurality_checkable_iteratorr   gslib.seek_ahead_threadr   gslib.sig_handlingr   r   gslib.storage_urlr   r   r   "gslib.third_party.storage_apitoolsr   r  gslib.utilsr   r   r   gslib.utils.boto_utilr   gslib.utils.cloud_api_helperr    gslib.utils.copy_helperr!   r"   r#   r$   gslib.utils.hashing_helperr%   r&   r'   r(   gslib.utils.metadata_utilr)   r*   r+   gslib.utils.posix_utilr,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   gslib.utils.rsync_utilr@   rA   gslib.utils.shim_utilrB   rC   gslib.utils.system_utilrD   gslib.utils.translation_helperrE   gslib.utils.unit_utilrF   rG   rH   gslib.wildcard_iteratorrI   PY3r4  r   r  r  rp   _OUTPUT_BUFFER_SIZEr   rS   rP   rL   rm   r|   r   r   r   r   r   r   r   objectr   rm  ry  r  r  r  r  rk  rQ   rO   <module>rC     s   1 & %  '    	   	 	     
 
    8 - , ! ) E 2 ( , 5 I 3 / 4 8 6 2 W ! # 2 6 < 8 @ 2 > L I @ : : F 9 - 9 5 N + > , - ( * * < = 2 J + ? 6 3 8 - 3 3 2 . = 5 1 ) :77	$	
[] @ 
 !&  

:
+4\$NF(M#`M `8&"4 nf6F f6R6V 6(+)= +)`@6F
+Y@7 Y@rQ   