170 lines
6.5 KiB
Python
170 lines
6.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2013 Google Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Additional help about CRC32C and installing crcmod."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
from __future__ import unicode_literals
|
|
|
|
from gslib.help_provider import HelpProvider
|
|
|
|
_DETAILED_HELP_TEXT = ("""
|
|
<B>OVERVIEW</B>
|
|
Google Cloud Storage provides a cyclic redundancy check (CRC) header that
|
|
allows clients to verify the integrity of object contents. For non-composite
|
|
objects Google Cloud Storage also provides an MD5 header to allow clients to
|
|
verify object integrity, but for composite objects only the CRC is available.
|
|
gsutil automatically performs integrity checks on all uploads and downloads.
|
|
Additionally, you can use the ``gsutil hash`` command to calculate a CRC for
|
|
any local file.
|
|
|
|
The CRC variant used by Google Cloud Storage is called CRC32C (Castagnoli),
|
|
which is not available in the standard Python distribution. The implementation
|
|
of CRC32C used by gsutil is provided by a third-party Python module called
|
|
`crcmod <https://pypi.python.org/pypi/crcmod>`_.
|
|
|
|
The crcmod module contains a pure-Python implementation of CRC32C, but using
|
|
it results in slow checksum computation and subsequently very poor
|
|
performance. A Python C extension is also provided by crcmod, which requires
|
|
compiling into a binary module for use. gsutil ships with a precompiled
|
|
crcmod C extension for macOS; for other platforms, see the installation
|
|
instructions below.
|
|
|
|
At the end of each copy operation, the ``gsutil cp``, ``gsutil mv``, and
|
|
``gsutil rsync`` commands validate that the checksum of the source
|
|
file/object matches the checksum of the destination file/object. If the
|
|
checksums do not match, gsutil will delete the invalid copy and print a
|
|
warning message. This very rarely happens, but if it does, you should
|
|
retry the operation.
|
|
|
|
|
|
<B>CONFIGURATION</B>
|
|
To determine if the compiled version of crcmod is available in your Python
|
|
environment, you can inspect the output of the ``gsutil version`` command for
|
|
the "compiled crcmod" entry:
|
|
|
|
$ gsutil version -l
|
|
...
|
|
compiled crcmod: True
|
|
...
|
|
|
|
If your crcmod library is compiled to a native binary, this value will be
|
|
True. If using the pure-Python version, the value will be False.
|
|
|
|
To control gsutil's behavior in response to crcmod's status, you can set the
|
|
``check_hashes`` variable in your `boto configuration file
|
|
<https://cloud.google.com/storage/docs/boto-gsutil>`_. For details on this
|
|
variable, see the surrounding comments in your boto configuration file. If
|
|
``check_hashes`` is not present in your configuration file, regenerate the
|
|
file by running ``gsutil config`` with the appropriate ``-e`` or ``-a`` flag.
|
|
|
|
|
|
<B>INSTALLATION</B>
|
|
These installation instructions assume that:
|
|
|
|
- You have ``pip`` installed. Consult the `pip installation instructions
|
|
<https://pip.pypa.io/en/stable/installing/>`_ for details on how
|
|
to install ``pip``.
|
|
- Your installation of ``pip`` can be found in your ``PATH`` environment
|
|
variable. If it cannot, you may need to replace ``pip3`` in the commands
|
|
below with the full path to the executable.
|
|
- You are installing the crcmod package for use with your system installation
|
|
of Python, and thus use the ``sudo`` command. If installing crcmod for a
|
|
different Python environment (e.g. in a virtualenv), you should omit
|
|
``sudo`` from the commands below.
|
|
- You are using a Python 3 version with gsutil. You can determine which
|
|
Python version gsutil is using by running ``gsutil version -l`` and looking
|
|
for the ``python version: 2.x.x`` or ``python version: 3.x.x`` line.
|
|
|
|
CentOS, RHEL, and Fedora
|
|
------------------------
|
|
|
|
To compile and install crcmod:
|
|
|
|
yum install gcc python3-devel python3-setuptools redhat-rpm-config
|
|
sudo pip3 uninstall crcmod
|
|
sudo pip3 install --no-cache-dir -U crcmod
|
|
|
|
Debian and Ubuntu
|
|
-----------------
|
|
|
|
To compile and install crcmod:
|
|
|
|
sudo apt-get install gcc python3-dev python3-setuptools
|
|
sudo pip3 uninstall crcmod
|
|
sudo pip3 install --no-cache-dir -U crcmod
|
|
|
|
Enterprise SUSE
|
|
-----------------
|
|
|
|
To compile and install crcmod when using Enterprise SUSE for SAP 12:
|
|
|
|
sudo zypper install gcc python-devel
|
|
sudo pip uninstall crcmod
|
|
sudo pip install --no-cache-dir -U crcmod
|
|
|
|
To compile and install crcmod when using Enterprise SUSE for SAP 15:
|
|
|
|
sudo zypper install gcc python3-devel
|
|
sudo pip uninstall crcmod
|
|
sudo pip install --no-cache-dir -U crcmod
|
|
|
|
macOS
|
|
-----
|
|
|
|
gsutil distributes a pre-compiled version of crcmod for macOS, so you shouldn't
|
|
need to compile and install it yourself. If for some reason the pre-compiled
|
|
version is not being detected, please let the Google Cloud Storage team know
|
|
(see ``gsutil help support``).
|
|
|
|
To compile manually on macOS, you will first need to install
|
|
`Xcode <https://developer.apple.com/xcode/>`_ and then run:
|
|
|
|
pip3 install -U crcmod
|
|
|
|
Windows
|
|
-------
|
|
|
|
An installer is available for the compiled version of crcmod from the Python
|
|
Package Index (PyPi) at the following URL:
|
|
|
|
https://pypi.python.org/pypi/crcmod/1.7
|
|
|
|
NOTE: If you have installed crcmod and gsutil hasn't detected it, it may have
|
|
been installed to the wrong directory. It should be located at
|
|
<python_dir>\\files\\Lib\\site-packages\\crcmod\\
|
|
|
|
In some cases the installer will incorrectly install to
|
|
<python_dir>\\Lib\\site-packages\\crcmod\\
|
|
|
|
Manually copying the crcmod directory to the correct location should resolve
|
|
the issue.
|
|
""")
|
|
|
|
|
|
class CommandOptions(HelpProvider):
|
|
"""Additional help about CRC32C and installing crcmod."""
|
|
|
|
# Help specification. See help_provider.py for documentation.
|
|
help_spec = HelpProvider.HelpSpec(
|
|
help_name='crc32c',
|
|
help_name_aliases=['crc32', 'crc', 'crcmod'],
|
|
help_type='additional_help',
|
|
help_one_line_summary='CRC32C and Installing crcmod',
|
|
help_text=_DETAILED_HELP_TEXT,
|
|
subcommand_help_text={},
|
|
)
|