
    (                        S r SSKJr  SSKJr  SSKJr  SSKJr  SSKrSSKrSSKrSSK	r	SSK
r
SSKrSSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SSK J!r!  SSK J"r"  SSK J#r#  SSK$J%r&  SSK'J(r(  SSK'J)r)  SSK'J*r*  SSK+J,r,  SSK-J.r.  SSK-J/r/  SSK0J1r1  SSK0J2r2  SSK0J3r3  SS K0J4r4  SS!K0J5r5  SS"K0J6r6  SS#K7J8r8  SS$K7J9r9  SS%K7J:r:  SS&K7J;r;  SS'K7J<r<  SS(K7J=r=  SS)K>J?r?  SS*K>J@r@  SS+KAJBrB  SS,KAJCrC  SS-KAJDrD  SS.KEJFrF  SS/KEJGrG  SS0KHJIrI  SS1KHJJrJ  S2rKS3\K-   rLS4rMS5rNS6rOS7rPS8rQS9rRS:rSS;rTS<rUS=rVS>rWS?rXS@rYSAR                  \L\M\N\O\P\Q\R\S\T\U\V\W\X\Y/5      r[SBr\0 SC\?" SD5      _SE\?" SF5      _SG\?" SH5      _SI\?" SJ5      _SK\?" SL5      _SM\?" SN5      _SO\?" SP5      _SQ\?" SR5      _SS\?" ST5      _SU\?" SV5      _SW\?" SX5      _SY\?" SZ5      _S[\?" S\5      _S]\?" S^5      _S_\?" S`5      _Sa\?" Sb5      _Sc\?" Sd5      _r]\^" \]R                  5       5      Se\?" Se5      4Sf\?" Se5      4/-    V Vs0 s H  u  pX_M	     snn r`Sg raSmSh jrbSi rcSj rd " Sk Sl\5      regs  snn f )nzCImplementation of Unix-like cp command for cloud storage providers.    )absolute_import)print_function)division)unicode_literalsN)encoding)gcs_json_api)Command)CommandArgument)ApiSelector)CommandException)LogPerformanceSummaryParams)CopyObjectsIterator)DestinationInfo)NameExpansionIterator)%NameExpansionIteratorDestinationTuple)SeekAheadNameExpansionIterator)ContainsWildcard)IsCloudSubdirPlaceholder)StorageUrlFromString)storage_v1_messages)
cat_helper)copy_helper)parallelism_framework_util)GetCloudApiInstance)DEBUGLEVEL_DUMP_REQUESTS)NO_MAX)CreateCopyHelperOpts)GetSourceFieldsNeededForCopy)GZIP_ALL_FILES)ItemExistsError)Manifest)SkipUnsupportedObjectError)ConvertModeToBase8)+DeserializeFileAttributesFromObjectMetadata)InitializePreservePosixData)POSIXAttributes)'SerializeFileAttributesToObjectMetadata)ValidateFilePermissionAccess)GcloudStorageFlag)GcloudStorageMap)GetStreamFromFileUrl)StdinIterator)StdinIteratorCls)NormalizeStorageClass)RemoveCRLFFromString)CalculateThroughput)MakeHumanReadablezw
  gsutil cp [OPTION]... src_url dst_url
  gsutil cp [OPTION]... src_url... dst_url
  gsutil cp [OPTION]... -I dst_url
z
<B>SYNOPSIS</B>
an  
<B>DESCRIPTION</B>
  The ``gsutil cp`` command allows you to copy data between your local file
  system and the cloud, within the cloud, and between
  cloud storage providers. For example, to upload all text files from the
  local directory to a bucket, you can run:

    gsutil cp *.txt gs://my-bucket

  You can also download data from a bucket. The following command downloads
  all text files from the top-level of a bucket to your current directory:

    gsutil cp gs://my-bucket/*.txt .

  You can use the ``-n`` option to prevent overwriting the content of
  existing files. The following example downloads text files from a bucket
  without clobbering the data in your directory:

    gsutil cp -n gs://my-bucket/*.txt .

  Use the ``-r`` option to copy an entire directory tree.
  For example, to upload the directory tree ``dir``:

    gsutil cp -r dir gs://my-bucket

  If you have a large number of files to transfer, you can perform a parallel
  multi-threaded/multi-processing copy using the
  top-level gsutil ``-m`` option (see "gsutil help options"):

    gsutil -m cp -r dir gs://my-bucket

  You can use the ``-I`` option with ``stdin`` to specify a list of URLs to
  copy, one per line. This allows you to use gsutil
  in a pipeline to upload or download objects as generated by a program:

    cat filelist | gsutil -m cp -I gs://my-bucket

  or:

    cat filelist | gsutil -m cp -I ./download_dir

  where the output of ``cat filelist`` is a list of files, cloud URLs, and
  wildcards of files and cloud URLs.

  NOTE: Shells like ``bash`` and ``zsh`` sometimes attempt to expand
  wildcards in ways that can be surprising. You may also encounter issues when
  attempting to copy files whose names contain wildcard characters. For more
  details about these issues, see `Wildcard behavior considerations
  <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_.
ah	  
<B>HOW NAMES ARE CONSTRUCTED</B>
  The ``gsutil cp`` command attempts to name objects in ways that are consistent with the
  Linux ``cp`` command. This means that names are constructed depending
  on whether you're performing a recursive directory copy or copying
  individually-named objects, or whether you're copying to an existing or
  non-existent directory.

  When you perform recursive directory copies, object names are constructed to
  mirror the source directory structure starting at the point of recursive
  processing. For example, if ``dir1/dir2`` contains the file ``a/b/c``, then the
  following command creates the object ``gs://my-bucket/dir2/a/b/c``:

    gsutil cp -r dir1/dir2 gs://my-bucket

  In contrast, copying individually-named files results in objects named by
  the final path component of the source files. For example, assuming again that
  ``dir1/dir2`` contains ``a/b/c``, the following command creates the object
  ``gs://my-bucket/c``:

    gsutil cp dir1/dir2/** gs://my-bucket

  Note that in the above example, the '**' wildcard matches all names
  anywhere under ``dir``. The wildcard '*' matches names just one level deep. For
  more details, see `URI wildcards
  <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_.

  The same rules apply for uploads and downloads: recursive copies of buckets and
  bucket subdirectories produce a mirrored filename structure, while copying
  individually or wildcard-named objects produce flatly-named files.

  In addition, the resulting names depend on whether the destination subdirectory
  exists. For example, if ``gs://my-bucket/subdir`` exists as a subdirectory,
  the following command creates the object ``gs://my-bucket/subdir/dir2/a/b/c``:

    gsutil cp -r dir1/dir2 gs://my-bucket/subdir

  In contrast, if ``gs://my-bucket/subdir`` does not exist, this same ``gsutil cp``
  command creates the object ``gs://my-bucket/subdir/a/b/c``.

  NOTE: The
  `Google Cloud Platform Console <https://console.cloud.google.com>`_
  creates folders by creating "placeholder" objects that end
  with a "/" character. gsutil skips these objects when downloading from the
  cloud to the local file system, because creating a file that
  ends with a "/" is not allowed on Linux and macOS. We
  recommend that you only create objects that end with "/" if you don't
  intend to download such objects using gsutil.
a  
<B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B>
  You can use gsutil to copy to and from subdirectories by using a command
  like this:

    gsutil cp -r dir gs://my-bucket/data

  This causes ``dir`` and all of its files and nested subdirectories to be
  copied under the specified destination, resulting in objects with names like
  ``gs://my-bucket/data/dir/a/b/c``. Similarly, you can download from bucket
  subdirectories using the following command:

    gsutil cp -r gs://my-bucket/data dir

  This causes everything nested under ``gs://my-bucket/data`` to be downloaded
  into ``dir``, resulting in files with names like ``dir/data/a/b/c``.

  Copying subdirectories is useful if you want to add data to an existing
  bucket directory structure over time. It's also useful if you want
  to parallelize uploads and downloads across multiple machines (potentially
  reducing overall transfer time compared with running ``gsutil -m
  cp`` on one machine). For example, if your bucket contains this structure:

    gs://my-bucket/data/result_set_01/
    gs://my-bucket/data/result_set_02/
    ...
    gs://my-bucket/data/result_set_99/

  you can perform concurrent downloads across 3 machines by running these
  commands on each machine, respectively:

    gsutil -m cp -r gs://my-bucket/data/result_set_[0-3]* dir
    gsutil -m cp -r gs://my-bucket/data/result_set_[4-6]* dir
    gsutil -m cp -r gs://my-bucket/data/result_set_[7-9]* dir

  Note that ``dir`` could be a local directory on each machine, or a
  directory mounted off of a shared file server. The performance of the latter
  depends on several factors, so we recommend experimenting
  to find out what works best for your computing environment.
a  
<B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B>
  If both the source and destination URL are cloud URLs from the same
  provider, gsutil copies data "in the cloud" (without downloading
  to and uploading from the machine where you run gsutil). In addition to
  the performance and cost advantages of doing this, copying in the cloud
  preserves metadata such as ``Content-Type`` and ``Cache-Control``. In contrast,
  when you download data from the cloud, it ends up in a file with
  no associated metadata, unless you have some way to keep
  or re-create that metadata.

  Copies spanning locations and/or storage classes cause data to be rewritten
  in the cloud, which may take some time (but is still faster than
  downloading and re-uploading). Such operations can be resumed with the same
  command if they are interrupted, so long as the command parameters are
  identical.

  Note that by default, the gsutil ``cp`` command does not copy the object
  ACL to the new object, and instead uses the default bucket ACL (see
  "gsutil help defacl"). You can override this behavior with the ``-p``
  option.

  When copying in the cloud, if the destination bucket has Object Versioning
  enabled, by default ``gsutil cp`` copies only live versions of the
  source object. For example, the following command causes only the single live
  version of ``gs://bucket1/obj`` to be copied to ``gs://bucket2``, even if there
  are noncurrent versions of ``gs://bucket1/obj``:

    gsutil cp gs://bucket1/obj gs://bucket2

  To also copy noncurrent versions, use the ``-A`` flag:

    gsutil cp -A gs://bucket1/obj gs://bucket2

  The top-level gsutil ``-m`` flag is  not allowed when using the ``cp -A`` flag.
z


<B>CHECKSUM VALIDATION</B>
  gsutil automatically performs checksum validation for copies to and from Cloud
  Storage. For more information, see `Hashes and ETags
  <https://cloud.google.com/storage/docs/hashes-etags#cli>`_.
a  
<B>RETRY HANDLING</B>
  The ``cp`` command retries when failures occur, but if enough failures happen
  during a particular copy or delete operation, or if a failure isn't retryable,
  the ``cp`` command skips that object and moves on. If any failures were not
  successfully retried by the end of the copy run, the ``cp`` command reports the
  number of failures and exits with a non-zero status.

  For details about gsutil's overall retry handling, see `Retry strategy
  <https://cloud.google.com/storage/docs/retry-strategy#tools>`_.
a  
<B>RESUMABLE TRANSFERS</B>
  gsutil automatically resumes interrupted downloads and interrupted `resumable
  uploads <https://cloud.google.com/storage/docs/resumable-uploads#gsutil>`_,
  except when performing streaming transfers. In the case of an interrupted
  download, a partially downloaded temporary file is visible in the destination
  directory with the suffix ``_.gstmp`` in its name. Upon completion, the
  original file is deleted and replaced with the downloaded contents.

  Resumable transfers store state information in files under
  ~/.gsutil, named by the destination object or file.

  See "gsutil help prod" for details on using resumable transfers
  in production.
a  
<B>STREAMING TRANSFERS</B>
  Use '-' in place of src_url or dst_url to perform a `streaming transfer
  <https://cloud.google.com/storage/docs/streaming>`_.

  Streaming uploads using the `JSON API
  <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ are buffered
  in memory part-way back into the file and can thus sometimes resume in the event
  of network or service problems.

  gsutil does not support resuming streaming uploads using the XML API or
  resuming streaming downloads for either JSON or XML. If you have a large amount
  of data to transfer in these cases, we recommend that you write the data to a
  local file and copy that file rather than streaming it.
a<  
<B>SLICED OBJECT DOWNLOADS</B>
  gsutil can automatically use ranged ``GET`` requests to perform downloads in
  parallel for large files being downloaded from Cloud Storage. See `sliced object
  download documentation
  <https://cloud.google.com/storage/docs/sliced-object-downloads>`_
  for a complete discussion.
a  
<B>PARALLEL COMPOSITE UPLOADS</B>
  gsutil can automatically use
  `object composition <https://cloud.google.com/storage/docs/composite-objects>`_
  to perform uploads in parallel for large, local files being uploaded to
  Cloud Storage. See the `parallel composite uploads documentation
  <https://cloud.google.com/storage/docs/parallel-composite-uploads>`_ for a
  complete discussion.
am  
<B>CHANGING TEMP DIRECTORIES</B>
  gsutil writes data to a temporary directory in several cases:

  - when compressing data to be uploaded (see the ``-z`` and ``-Z`` options)
  - when decompressing data being downloaded (for example, when the data has
    ``Content-Encoding:gzip`` as a result of being uploaded
    using gsutil cp -z or gsutil cp -Z)
  - when running integration tests using the gsutil test command

  In these cases, it's possible the temporary file location on your system that
  gsutil selects by default may not have enough space. If gsutil runs out of
  space during one of these operations (for example, raising
  "CommandException: Inadequate temp space available to compress <your file>"
  during a ``gsutil cp -z`` operation), you can change where it writes these
  temp files by setting the TMPDIR environment variable. On Linux and macOS,
  you can set the variable as follows:

    TMPDIR=/some/directory gsutil cp ...

  You can also add this line to your ~/.bashrc file and restart the shell
  before running gsutil:

    export TMPDIR=/some/directory

  On Windows 7, you can change the TMPDIR environment variable from Start ->
  Computer -> System -> Advanced System Settings -> Environment Variables.
  You need to reboot after making this change for it to take effect. Rebooting
  is not necessary after running the export command on Linux and macOS.
a  
<B>SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SUCH AS SYMLINKS AND DEVICES)</B>

  Please see the section about OS-specific file types in "gsutil help rsync".
  While that section refers to the ``rsync`` command, analogous
  points apply to the ``cp`` command.
aT2  
<B>OPTIONS</B>
  -a predef_acl  Applies the specific predefined ACL to uploaded objects. See
                 "gsutil help acls" for further details.

  -A             Copy all source versions from a source bucket or folder.
                 If not set, only the live version of each source object is
                 copied.

                 NOTE: This option is only useful when the destination
                 bucket has Object Versioning enabled. Additionally, the generation
                 numbers of copied versions do not necessarily match the order of the
                 original generation numbers.

  -c             If an error occurs, continue attempting to copy the remaining
                 files. If any copies are unsuccessful, gsutil's exit status
                 is non-zero, even if this flag is set. This option is
                 implicitly set when running ``gsutil -m cp...``.

                 NOTE: ``-c`` only applies to the actual copying operation. If an
                 error, such as ``invalid Unicode file name``, occurs while iterating
                 over the files in the local directory, gsutil prints an error
                 message and aborts.

  -D             Copy in "daisy chain" mode, which means copying between two buckets
                 by first downloading to the machine where gsutil is run, then
                 uploading to the destination bucket. The default mode is a
                 "copy in the cloud," where data is copied between two buckets without
                 uploading or downloading.

                 During a "copy in the cloud," a source composite object remains composite
                 at its destination. However, you can use "daisy chain" mode to change a
                 composite object into a non-composite object. For example:

                     gsutil cp -D gs://bucket/obj gs://bucket/obj_tmp
                     gsutil mv gs://bucket/obj_tmp gs://bucket/obj

                 NOTE: "Daisy chain" mode is automatically used when copying
                 between providers: for example, when copying data from Cloud Storage
                 to another provider.

  -e             Exclude symlinks. When specified, symbolic links are not copied.

  -I             Use ``stdin`` to specify a list of files or objects to copy. You can use
                 gsutil in a pipeline to upload or download objects as generated by a program.
                 For example:

                   cat filelist | gsutil -m cp -I gs://my-bucket

                 where the output of ``cat filelist`` is a one-per-line list of
                 files, cloud URLs, and wildcards of files and cloud URLs.

  -j <ext,...>   Applies gzip transport encoding to any file upload whose
                 extension matches the ``-j`` extension list. This is useful when
                 uploading files with compressible content such as .js, .css,
                 or .html files. This also saves network bandwidth while
                 leaving the data uncompressed in Cloud Storage.

                 When you specify the ``-j`` option, files being uploaded are
                 compressed in-memory and on-the-wire only. Both the local
                 files and Cloud Storage objects remain uncompressed. The
                 uploaded objects retain the ``Content-Type`` and name of the
                 original files.

                 Note that if you want to use the ``-m`` `top-level option
                 <https://cloud.google.com/storage/docs/gsutil/addlhelp/GlobalCommandLineOptions>`_
                 to parallelize copies along with the ``-j/-J`` options, your
                 performance may be bottlenecked by the
                 "max_upload_compression_buffer_size" boto config option,
                 which is set to 2 GiB by default. You can change this
                 compression buffer size to a higher limit. For example:

                   gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \
                     -m cp -j html,txt -r /local/source/dir gs://bucket/path

  -J             Applies gzip transport encoding to file uploads. This option
                 works like the ``-j`` option described above, but it applies to
                 all uploaded files, regardless of extension.

                 CAUTION: If some of the source files don't compress well, such
                 as binary data, using this option may result in longer uploads.

  -L <file>      Outputs a manifest log file with detailed information about
                 each item that was copied. This manifest contains the following
                 information for each item:

                 - Source path.
                 - Destination path.
                 - Source size.
                 - Bytes transferred.
                 - MD5 hash.
                 - Transfer start time and date in UTC and ISO 8601 format.
                 - Transfer completion time and date in UTC and ISO 8601 format.
                 - Upload id, if a resumable upload was performed.
                 - Final result of the attempted transfer, either success or failure.
                 - Failure details, if any.

                 If the log file already exists, gsutil uses the file as an
                 input to the copy process, and appends log items to
                 the existing file. Objects that are marked in the
                 existing log file as having been successfully copied or
                 skipped are ignored. Objects without entries are
                 copied and ones previously marked as unsuccessful are
                 retried. This option can be used in conjunction with the ``-c`` option to
                 build a script that copies a large number of objects reliably,
                 using a bash script like the following:

                   until gsutil cp -c -L cp.log -r ./dir gs://bucket; do
                     sleep 1
                   done

                 The -c option enables copying to continue after failures
                 occur, and the -L option allows gsutil to pick up where it
                 left off without duplicating work. The loop continues
                 running as long as gsutil exits with a non-zero status. A non-zero
                 status indicates there was at least one failure during the copy
                 operation.

                 NOTE: If you are synchronizing the contents of a
                 directory and a bucket, or the contents of two buckets, see
                 "gsutil help rsync".

  -n             No-clobber. When specified, existing files or objects at the
                 destination are not replaced. Any items that are skipped
                 by this option are reported as skipped. gsutil
                 performs an additional GET request to check if an item
                 exists before attempting to upload the data. This saves gsutil
                 from retransmitting data, but the additional HTTP requests may make
                 small object transfers slower and more expensive.

  -p             Preserves ACLs when copying in the cloud. Note
                 that this option has performance and cost implications only when
                 using the XML API, as the XML API requires separate HTTP calls for
                 interacting with ACLs. You can mitigate this
                 performance issue using ``gsutil -m cp`` to perform parallel
                 copying. Note that this option only works if you have OWNER access
                 to all objects that are copied. If you want all objects in the
                 destination bucket to end up with the same ACL, you can avoid these
                 performance issues by setting a default object ACL on that bucket
                 instead of using ``cp -p``. See "gsutil help defacl".

                 Note that it's not valid to specify both the ``-a`` and ``-p`` options
                 together.

  -P             Enables POSIX attributes to be preserved when objects are
                 copied. ``gsutil cp`` copies fields provided by ``stat``. These fields
                 are the user ID of the owner, the group
                 ID of the owning group, the mode or permissions of the file, and
                 the access and modification time of the file. For downloads, these
                 attributes are only set if the source objects were uploaded
                 with this flag enabled.

                 On Windows, this flag only sets and restores access time and
                 modification time. This is because Windows doesn't support
                 POSIX uid/gid/mode.

  -R, -r         The ``-R`` and ``-r`` options are synonymous. They enable directories,
                 buckets, and bucket subdirectories to be copied recursively.
                 If you don't use this option for an upload, gsutil copies objects
                 it finds and skips directories. Similarly, if you don't
                 specify this option for a download, gsutil copies
                 objects at the current bucket directory level and skips subdirectories.

  -s <class>     Specifies the storage class of the destination object. If not
                 specified, the default storage class of the destination bucket
                 is used. This option is not valid for copying to non-cloud destinations.

  -U             Skips objects with unsupported object types instead of failing.
                 Unsupported object types include Amazon S3 objects in the GLACIER
                 storage class.

  -v             Prints the version-specific URL for each uploaded object. You can
                 use these URLs to safely make concurrent upload requests, because
                 Cloud Storage refuses to perform an update if the current
                 object version doesn't match the version-specific URL. See
                 `generation numbers
                 <https://cloud.google.com/storage/docs/metadata#generation-number>`_
                 for more details.

  -z <ext,...>   Applies gzip content-encoding to any file upload whose
                 extension matches the ``-z`` extension list. This is useful when
                 uploading files with compressible content such as .js, .css,
                 or .html files, because it reduces network bandwidth and storage
                 sizes. This can both improve performance and reduce costs.

                 When you specify the ``-z`` option, the data from your files is
                 compressed before it is uploaded, but your actual files are
                 left uncompressed on the local disk. The uploaded objects
                 retain the ``Content-Type`` and name of the original files, but
                 have their ``Content-Encoding`` metadata set to ``gzip`` to
                 indicate that the object data stored are compressed on the
                 Cloud Storage servers and have their ``Cache-Control`` metadata
                 set to ``no-transform``.

                 For example, the following command:

                   gsutil cp -z html \
                     cattypes.html tabby.jpeg gs://mycats

                 does the following:

                 - The ``cp`` command uploads the files ``cattypes.html`` and
                   ``tabby.jpeg`` to the bucket ``gs://mycats``.
                 - Based on the file extensions, gsutil sets the ``Content-Type``
                   of ``cattypes.html`` to ``text/html`` and ``tabby.jpeg`` to
                   ``image/jpeg``.
                 - The ``-z`` option compresses the data in the file ``cattypes.html``.
                 - The ``-z`` option also sets the ``Content-Encoding`` for
                   ``cattypes.html`` to ``gzip`` and the ``Cache-Control`` for
                   ``cattypes.html`` to ``no-transform``.

                 Because the ``-z/-Z`` options compress data prior to upload, they
                 are not subject to the same compression buffer bottleneck that
                 can affect the ``-j/-J`` options.

                 Note that if you download an object with ``Content-Encoding:gzip``,
                 gsutil decompresses the content before writing the local file.

  -Z             Applies gzip content-encoding to file uploads. This option
                 works like the ``-z`` option described above, but it applies to
                 all uploaded files, regardless of extension.

                 CAUTION: If some of the source files don't compress well, such
                 as binary data, using this option may result in files taking up
                 more space in the cloud than they would if left uncompressed.

  --stet         If the STET binary can be found in boto or PATH, cp will
                 use the split-trust encryption tool for end-to-end encryption.
z

za:AcDeIL:MNnpPrRs:tUvz:Zj:J-Az--all-versions-az--predefined-acl-cz--continue-on-error-Dz--daisy-chain-ez--ignore-symlinks-Iz--read-paths-from-stdin-Jz--gzip-in-flight-all-jz--gzip-in-flight-Lz--manifest-path-nz--no-clobber-Pz--preserve-posix-pz--preserve-acl-sz--storage-class-vz--print-created-message-Zz--gzip-local-all-zz--gzip-local-Uz--skip-unsupported-r-Rc                     Sn[        U 5       H  u  nu  p4US:X  d  M  Un  O   Ub1  U W   S   nX   S   [        R                  R                  XU5      4X'   gg)aL  Gcloud uses camel-case predefined/canned ACLs, and gsutil uses snake-case.

The camel-case-snake-case difference is related to gcloud primarily using
JSON API rather than the XML API.

Predefined ACLs are also called "canned ACLs".

Args:
  sub_opts: List of pairs representing flag keys and values, e.g.
    [('a', 'public-read')]
Nr3      r   )	enumerater   +FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATIONget)sub_optspredefined_acl_idxik_old_predefined_acls         $platform/gsutil/gslib/commands/cp.py'ShimTranslatePredefinedAclSubOptForCopyrQ     st     X&ia!Dy ' #!!Q;q>KKOO)?@HK $    c                 :    U R                  UUU R                  S9  g )N)thread_statepreserve_posix)CopyFuncpreserve_posix_attrs)clsargsrT   s      rP   _CopyFuncWrapperrZ     s!    ,,t(!66  8rR   c                     U R                   R                  [        U5      5        U =R                  S-  sl        U R                   R	                  S[
        R                  " 5       5        g)9Simple exception handler to allow post-completion status.rF   z*

Encountered exception while copying:
%s
N)loggererrorstrop_failure_countdebug	traceback
format_excrX   es     rP   _CopyExceptionHandlerrf     sJ    **3q6!**C'')+rR   c                 L    U R                   R                  [        U5      5        g)r\   N)r]   r^   r_   rd   s     rP   _RmExceptionHandlerrh     s    **3q6rR   c                     ^  \ rS rSrSr\R                  " SS/\S\\	SSS\
R                  \
R                  /\
R                  S	S
/\R                  " 5       /S9r\R                   " SS/SS\0 S9rU 4S jrSS jrS rS rS rSrU =r$ )	CpCommandi  a2  Implementation of gsutil cp command.

Note that CpCommand is run for both gsutil cp and gsutil mv. The latter
happens by MvCommand calling CpCommand and passing the hidden (undocumented)
-M option. This allows the copy and remove needed for each mv to run
together (rather than first running all the cp's and then all the rm's, as
we originally had implemented), which in turn avoids the following problem
with removing the wrong objects: starting with a bucket containing only
the object gs://bucket/obj, say the user does:
  gsutil mv gs://bucket/* gs://bucket/d.txt
If we ran all the cp's and then all the rm's and we didn't expand the wildcard
first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt,
and the rm command would then remove that object. In the implementation
prior to gsutil release 3.12 we avoided this by building a list of objects
to process and then running the copies and then the removes; but building
the list up front limits scalability (compared with the current approach
of processing the bucket listing iterator on the fly).
cpcopyrF   TFr   stetztestcallbackfile=)command_name_aliasesusage_synopsismin_argsmax_argssupported_sub_argsfile_url_okprovider_url_okurls_start_arggs_api_supportgs_default_apisupported_private_argsargparse_argumentscommand_helpzCopy files and objects)	help_namehelp_name_aliases	help_typehelp_one_line_summary	help_textsubcommand_help_textc                    > U R                   R                  S5        [        U R                  5        [	        SS/[
        S9n[        TU ]  U5      $ )NzUnlike pure gsutil, this shim won't run composite uploads and sliced downloads in parallel by default. Use the -m flag to enable parallelism (i.e. "gsutil -m cp ...").storagerk   )gcloud_commandflag_map)r]   warnrQ   rJ   r*   CP_SHIM_FLAG_MAPsuperget_gcloud_storage_args)selfgcloud_storage_map	__class__s     rP   r   !CpCommand.get_gcloud_storage_args  sQ    KK	23 ,DMM:)!4(! 7*+=>>rR   c                 j   [        XS9n[        R                  " 5       nUR                  (       a  SnOU R                  nUR
                  nUR                  nUR                  n	UR                  n
UR                  5       (       a)  UR                  5       (       a  [        SU< SU< S35      eU(       a5  UR                  5       (       a   UR                  5       (       a  [        S5      eU R                  (       a5  UR                  5       (       a   UR                  5       (       a  [        S5      eU
(       a,  [        R                  " UR                   UR"                  U5        [%        U5      (       a  gUR&                  (       a+  U R(                  R+                  UR,                  5      (       a  gUR                  (       a  UR                  (       a  S	U l        UR                   R                  5       (       aj  [0        R2                  R5                  UR                   R6                  5      (       d2  U
(       a+   [0        R8                  " UR                   R6                  5        [        R@                  " UUU	U
URB                  UR                   UR"                  U R.                  US
9	n[        RD                  " X|5      n[        RF                  " X5        [        RH                  " X5      (       a  [        U< SU< SU< S35      eUR                  5       (       a(  URK                  5       (       a  [        U< SU< S35      eUR                  5       (       d  URL                  (       a  [        SU-  5      eSnURN                  (       a/  [P        RR                  " [T        RV                  URN                  5      nUR                  5       (       a  U(       a  U(       d  [T        RV                  " 5       n[0        RX                  " UR6                  5      u
  n    nnnnnnn[[        U5      n[]        UUUUUS9n[T        RV                  R_                  / S9n[a        UUUS
9  UUl1        U(       a  UR                  5       (       a  [e        XR,                  5      nURf                  Rh                  n[k        UR,                  URl                  URn                  US9u  nnU(       a6  U(       d/  [p        Rr                  " 5       Ru                  U5        [        S5      eSn UR&                  (       a0  U R(                  Rw                  UR,                  UR,                  5        U R.                  (       Ga  UR                   R6                  (       Ga  UR                  5       (       a  [0        R2                  Ry                  UR                   R6                  5      n[0        R2                  R{                  U[0        R2                  Ry                  UR6                  5      /5      R}                  U5      (       d^  U R~                  R                  SU< SU< S35        UR&                  (       a(  U R(                  R                  UR,                  SSS5        g[        R                  " U R~                  UUUU [        US	U R                  U R(                  U R                  U R                  UU R                  S9u  nnnnUR&                  (       aU  U(       a'  U R(                  R                  UR,                  SU5        U R(                  R                  UR,                  US5        UR                  (       a  U R~                  R                  SU5        UR                  (       a  U R~                  R                  SU5        UR                  5       (       a;  UR                  UR                  UR6                  UR                  UR                  S 9  O [0        R                  " UR6                  5        U R                     U =R                  U-  sl]        SSS5        g! [:         a*  nUR<                  [<        R>                  :w  a  e  SnAGNSnAff = f! [         a\    S!U-  nU R~                  R                  U5        UR&                  (       a(  U R(                  R                  UR,                  SSU5         N[         a}  nS"UR,                  < S#UR                  < 3nU R~                  R                  U5        UR&                  (       a(  U R(                  R                  UR,                  SSU5         SnAGNHSnAf[        R                   a/  nU R~                  R                  SU< S$U< S%35         SnAGNSnAf[         Gao  nUR                  (       az  [        R                  " U5      (       a_  S&U-  nU R~                  R                  U5        UR&                  (       a(  U R(                  R                  UR,                  SSU5         SnAGN"U R                  (       a  S'U< S([        U5      < 3nU =R                  S)-  slY        U R~                  R                  U5        UR&                  (       a1  U R(                  R                  UR,                  SS*[        U5      5         SnAGNUR&                  (       a1  U R(                  R                  UR,                  SS*[        U5      5        e SnAff = f! , (       d  f       g= f)+z@Worker function for performing the actual copy (and rm, for mv).)rT   mvzThe z3 command does not allow provider-only source URLs ()z/Cannot preserve POSIX attributes with a stream.z8Cannot upload from a stream when using gsutil -m option.NT)rU   z: "z" and "z" are the same file - abort.z: a version-specific URL
(z2)
cannot be the destination for gsutil cp - abort.z<Cannot specify storage class for a non-cloud destination: %s)atimemtimeuidgidmode)additionalProperties)r   r   r   zPThis sync will orphan file(s), please fix their permissions before trying again.r   zSkipping copy of source URL zH because it would be copied outside the expected destination directory: .skipz4Would have copied outside the destination directory.)src_obj_metadataallow_splittingheadersmanifestgzip_encoded	gzip_extsrU   use_stetmd5OKzCreated: %szRemoving %s...)
generationproviderzSkipping existing item: %szSkipping item z with unsupported object type z because destination URL zj is already being copied by another gsutil process or thread (did you specify the same source URL twice?) zRejected (noclobber): %szError copying z: rF   r^   )^r   r   GetCopyHelperOpts
perform_mvcommand_namesource_storage_urlexpanded_storage_urlnames_containeris_multi_source_request
IsCloudUrl
IsProviderr   	IsFileUrlIsStreamparallel_operationsInsistDstUrlNamesContainerexp_dst_urlhave_existing_dst_containerr   use_manifestr   WasSuccessful
url_stringrecursion_requestedospathexistsobject_namemakedirsOSErrorerrnoEEXISTConstructDstUrl!is_multi_top_level_source_requestFixWindowsNamingCheckForDirFileConflict
SrcDstSameHasGenerationdest_storage_classexpanded_resultr   JsonToMessageapitools_messagesObjectstatr#   r&   MetadataValuer'   metadatar$   r   permissionsr(   r   r   logging	getLoggercritical
Initializeabspath
commonpath
startswithr]   r   	SetResultPerformCopyrf   r   r   r   r   Set	print_verinfoDeleteObjectbucket_namer   schemeunlinkr    r"   unsupported_typeFileConcurrencySkipError	Exception
no_clobberIsNoClobberServerExceptioncontinue_on_errorr_   r`   r^   r/   
stats_locktotal_bytes_transferred)r   copy_object_inforT   rU   
gsutil_apicopy_helper_optscmd_namesrc_urlexp_src_urlsrc_url_names_containerhave_multiple_srcsre   dst_urlr   r   rN   r   r   r   r   posix_attrscustom_metadatavaliderrbytes_transferred	container
result_urlr   messages                                rP   rV   CpCommand.CopyFunc  s   $TEJ"446""h""h11G"77K.>>)AA 2 2 4 4W  '++--'2B2B2D2DNOOG$5$5$7$7G<L<L<N<N
DF F,,

&
&

6
6B  ,, $$)D)D*  * ""'7'G'G "&d$$..00GGNN+77CCDD
$00<<=
 ))::$$44  %	'G **7<G''=k33)1;I J J  5 5 7 7&1 2 2 $4$G$G /189 : : ''!//

"
"$4$D$DF ~,33546GG

!
!5#1dAq!S#q%%d#%*/(+(+)-	/k
 *00>>! ? #o-k.==KM #2G--//?
..0k))d/0B0B4?OO4?OO59;jeS 
$$S)  C D 	D c-		&	&  !7!79K9KL

"
"
"

&
&
2
2
2w7H7H7J7J
 GGOO$4$@$@$L$LM	 ww!!rwww':':;#
 :i ! ++

 I'( **MM##&&6FH .9.E.E
++




+,,==((NN'==/"+a	J 
	&	&
--

K22E3
? 6 68I4P		#	# 	
3J 
	$	$);7!!##

!
!+"9"9"-"9"9-8-C-C+6+=+= " ?
 ))K++
,	 ""&77" 
Q  77ell"
 #t  L,w6g
kkw		&	& 6 667K% L((!*<*<>g
kkw		&	& 6 667K// 
kk G 
  

%
%

0
0
3
3,w6!((
--
!
!+"8"8!VW
M!!!,3SV<"'"((
--
!
!+"8"8!W"6w"?A ((
--
!
!+"8"8!Wc!f
M#: 
ss   7*a E0a= C9a= 'l$
a:a55a:=A#l!"	l!+A2e##l!:$f$$l!2Bl=BlAll!$
l2c              #     #    [         R                  " 5       n[        X5       GH  u  pE[         R                  " UU R                  U R
                  U R                  S9u  pg[        [        U R                  U R                  U R                  U R                  UU R                  =(       d    UR                  U R
                  U R                  U R                  U R                  =(       d    U R                   [#        UR%                  5       UR&                  UR(                  U R*                  UR                  U R,                  S9S9[/        Xg5      5      nU R0                  =(       d    UR3                  5       U l        U R4                  =(       d    UR%                  5       U l        U R6                  R9                  UR:                  5        [<        R>                  " U R@                  U5      U l         Uv   GM     g 7f)N)r]   )rU   delete_sourcefile_size_will_change)
project_idall_versionsignore_symlinksr   bucket_listing_fields)!r   r   zipExpandUrlToSingleBlrr   r   r]   r   r   r   ra   r   r   r   exclude_symlinksr   r   r   r   skip_unsupported_objectspreserve_aclrW   r   r   has_file_dstr   has_cloud_dstprovider_typesaddr   	itertoolschaincombined_src_urls)	r   src_url_strs_iterdst_url_strsr   src_url_strdst_url_strr   r   !name_expansion_iterator_dst_tuples	            rP   /_ConstructNameExpansionIteratorDstTupleIterator9CpCommand._ConstructNameExpansionIteratorDstTupleIterator  s    "446$'(9$H 4 
*
*;+/??+/??26++? /k
 +P
jjkkoo&&E*:*E*E,,"33!%!7!7 ":!%!9!9$@((*";;"//!%!:!: 0 ; ;(,%78& +
C)+E', ++F{/D/D/Fd--I1G1G1Id
k001(t/E/E/: <d .-u %Is   G5G7c                    U R                  5       nSU l        [        U R                  S   5      nUR	                  5       (       a  UR
                  S:X  d  UR                  5       (       an  U R                  (       a  [        S5      eUR                  5       (       a
  [        USS9OS n[        R                  " U 5      R                  U R                  S S US9$ UR                  (       a0  [        U R                  5      S:w  a  [        S	5      e[        5       /nO4[        U R                  5      S
:  a  [        S5      eU R                  S S /nU R                  S   /n/ U l        SU l        SU l        ['        5       U l        [+        U R-                  XE5      UR.                  5      nU R1                  S S S SS9u  px[2        R4                  " X R6                  Xx-  5        S n	UR                  (       d  [9        U R:                  U R<                  U R?                  5       U R                   U R@                  =(       d    URB                  U RD                  U RF                  U RH                  U RJ                  S9	n	[L        RN                  " 5       U l(        SU l)        [T        RT                  " 5       n
SnU RW                  [X        U[Z        UU R\                  (       + U	S9  U R^                  R=                  SU R                  5        [T        RT                  " 5       nX-
  U l0        [c        U R                  U R`                  5      U l2        [g        U R"                  U R$                  U Rd                  U R                  U R`                  U Rh                  UR.                  [k        U R(                  5      S9  U R<                  [l        :  aU  U R                  S:w  aE  U R^                  Ro                  SU R                  U R`                  [q        U Rd                  5      5        U RR                  (       a7  U RR                  S:  a  SOSn[        SRs                  U RR                  US95      eg)Nr   -z?Cannot preserve POSIX attributes with a stream or a named pipe.wb)r   )
cat_out_fdrF   z.Source URLs cannot be specified with -I option   z+Wrong number of arguments for "cp" command.F)process_countthread_countparallel_operations_overrideprint_macos_warning)r   r   r   r   )r`   r   )fail_on_errorseek_ahead_iteratorztotal_bytes_transferred: %d)r  r  avg_throughputr   total_elapsed_timeuses_fanis_daisy_chainr  z;Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)s z5{count} file{pl}/object{pl} could not be transferred.)countpl):
_ParseOptsr   r   rY   r   r   IsFiforW   r   r+   r   	CatHelperCatUrlStringsread_args_from_stdinlenr-   r	  r  r  setr  r   r  daisy_chain_GetProcessAndThreadCountr   .TriggerReauthForDestinationProviderIfNecessaryr   r   r   ra   GetSeekAheadGsutilApir   r   r   r   r   r   r   
CreateLockr   r`   timeApplyrZ   rf   r   r]   r  r0   total_bytes_per_secondr   r   listr   r   r1   format)r   r   r   r  src_url_strsr  name_expansion_iteratorr  r  r  
start_timeshared_attrsend_time
plural_strs                 rP   
RunCommandCpCommand.RunCommand  s   (#$D "499R=1G 3 3s :gnn>N>N		"	"  9 : 	: ~~'' )t<-1 !!$'55diinAK 6 M M ,,	TYY1	OPP '()l	TYY!	LMMiin%lIIbM?LDDD%D 2<<	($$ #'"@"@%)!	 #A ##M
 >>-">@  00:



**

$
$
&

 
 

"
"
A&6&A&A((__// $	/ 1;;=DO D J CL
 	JJ&$"&"8"88#6  8 	KK3224 yy{H&3D"5$$d&=&=#?D&&((22 $ < <22))'33D//02 zz-- 
	%	%	*I(($*A*Ad99:	< //!33j 3396'+'<'< 4: 4MN N rR   c                    SnSU l         SnSU l        SnSnSnSnSnSU l        S nS U l        SU l        SU l        Sn	Sn
S nS nS nS nSU l        S U l        U R                  (       Ga  U R                   GH  u  nnUS:X  a	  UnSU l        US:X  a  SU l        US:X  a	  SU l        M2  US:X  a  SnM<  US:X  a	  SU l         MK  US:X  a  UnMU  US	:X  a  SnM_  US
:X  a3  Sn	UR                  S5       Vs/ s H  nUR                  5       PM     nnM  US:X  a
  Sn	[        nM  US:X  a  Sn[        U5      U l        M  US:X  a  SnM  US:X  a  SnM  US:X  a  SnM  US:X  a  SU l        [        5         M  US:X  d  US:X  a
  SU l        GM  US:X  a  [        U5      nGM#  US:X  a
  SU l        GM3  US:X  a  SnGM>  US:X  a4  Sn
UR                  S5       Vs/ s H  nUR                  5       PM     nnGMx  US:X  a  Sn
[        nGM  US:X  d  GM  SU l        GM     U(       a  U(       a  [!        S5      eU R                  (       a  U R"                  (       a  [!        S5      eU	(       a  U
(       a  [!        S5      eU(       a$  U(       a  U	(       a  [!        S5      e[!        S5      eU=(       d    UU l        Xl        [)        UUUUUUUUU R
                  UUS9$ s  snf s  snf ) NFr3   Tr2   r4   r5   r6   z--testcallbackfiler7   r9   ,r8   r:   z-Mr;   r=   r<   rC   rD   r>   rB   r?   rA   r@   z--stetz:Specifying both the -p and -a options together is invalid.zThe gsutil -m option is not supported with the cp -A flag, to ensure that object version ordering is preserved. Please re-run the command without the -m option.z@Specifying both the -j/-J and -z/-Z options together is invalid.z:Specifying both the -j and -J options together is invalid.z:Specifying both the -z and -Z options together is invalid.)r   r   r,  r)  r   r   r  
canned_aclr  test_callback_filer   )r   r   rW   cannedr   r  r   r   rJ   splitstripr   r!   r%   r   r.   r   r   r   r   r   )r   r   r   r,  r)  r   r   r  r@  r   
gzip_localgzip_arg_extsgzip_arg_allrA  r   oaxs                     rP   r%  CpCommand._ParseOpts  s   J "DJ #DK ILL %DJ DKD$)D! LJMLDM DM}}}--$!Q9*$+9"$
9#'$
 $Y+$Y"&$
&&  !
$Y!%
$Y,./ggcl;l1779l-;-$Y,',$Y,"1+$-$Y
 *$Y*$Y,$Y&*$
#
%
'$Y!t)%)$
"$Y4Q7
$Y*.$
'$Y)$Y*./ggcl;l1779l-;-$Y*',(]$-o  r 

FH H T55/0 0 

LN N	HJ 	J HJ 	J"2lDN$1!!!%!>!>--/ /w << <s   -KK)r   rB  r	  r   r   r   r   r  r  r   r`   rW   r  r   r  r   r3  r   r  r   )NF)__name__
__module____qualname____firstlineno____doc__r	   CreateCommandSpec	_SYNOPSISr   CP_SUB_ARGSr   XMLJSONr
   %MakeZeroOrMoreCloudOrFileURLsArgumentcommand_specHelpSpec_DETAILED_HELP_TEXT	help_specr   rV   r  r<  r%  __static_attributes____classcell__)r   s   @rP   rj   rj     s    ( **
"8$!oo{'7'78 %% %&9:

?
?
A!,* 4#)
?f8P=.@vpF/ F/rR   rj   )N)frP  
__future__r   r   r   r   r   r  r   r   r1  rb   apitools.base.pyr   gslibr   gslib.commandr	   gslib.command_argumentr
   gslib.cs_api_mapr   gslib.exceptionr   gslib.metricsr   gslib.name_expansionr   r   r   r   r   gslib.storage_urlr   r   r   "gslib.third_party.storage_apitoolsr   r   gslib.utilsr   r   r   gslib.utils.cloud_api_helperr   gslib.utils.constantsr   r   gslib.utils.copy_helperr   r   r   r    r!   r"   gslib.utils.posix_utilr#   r$   r%   r&   r'   r(   gslib.utils.shim_utilr)   r*   gslib.utils.system_utilr+   r,   r-   gslib.utils.text_utilr.   r/   gslib.utils.unit_utilr0   r1   rR  _SYNOPSIS_TEXT_DESCRIPTION_TEXT_NAME_CONSTRUCTION_TEXT_SUBDIRECTORIES_TEXT_COPY_IN_CLOUD_TEXT_CHECKSUM_VALIDATION_TEXT_RETRY_HANDLING_TEXT_RESUMABLE_TRANSFERS_TEXT_STREAMING_TRANSFERS_TEXT_SLICED_OBJECT_DOWNLOADS_TEXT _PARALLEL_COMPOSITE_UPLOADS_TEXT_CHANGING_TEMP_DIRECTORIES_TEXT_COPYING_SPECIAL_FILES_TEXT_OPTIONS_TEXTjoinrY  rS  CP_AND_MV_SHIM_FLAG_MAPr4  itemsr   rQ   rZ   rf   rh   rj   )rM   vs   00rP   <module>r     s;    J & %  '    	   %  ! 2 ( , 5 4 0 6 F ? . 6 2 W " # 2 < : ( 8 @ 2 3 , > 5 N > 2 J ? 3 2 8 1 4 7 6 5 3	1 f0 d' R# J 
     ! $  # > dL kk!$##  " ,
,
-
.
/ 	
1
2 	
O
,	
 	
/
0 	
5
6 	
2
3 	
.
/ 	
-
. 	
N
+ 	
.
/ 	
,
- 	
-
. 	
5
6 	
.
/  	
N
+!" 	
0
1# * 1779:
d#$t->t-D&EFGGTQAD G @08+
f	/ f	/cs    I,