# -*- coding: utf-8 -*- # # Copyright 2016 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Move local source snapshots to GCP.""" from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals import os import os.path import tarfile import zipfile from googlecloudsdk.api_lib.cloudbuild import metric_names from googlecloudsdk.api_lib.storage import storage_util from googlecloudsdk.command_lib.util import gcloudignore from googlecloudsdk.core import log from googlecloudsdk.core import metrics from googlecloudsdk.core.util import files _IGNORED_FILE_MESSAGE = """\ Some files were not included in the source upload. Check the gcloud log [{log_file}] to see which files and the contents of the default gcloudignore file used (see `$ gcloud topic gcloudignore` to learn more). """ def _ResetOwnership(tarinfo): tarinfo.uid = tarinfo.gid = 0 tarinfo.uname = tarinfo.gname = 'root' return tarinfo class Snapshot(storage_util.Snapshot): """Snapshot is a manifest of the source in a directory. """ def _MakeTarball(self, archive_path): """Constructs a tarball of snapshot contents. Args: archive_path: Path to place tar file. Returns: tarfile.TarFile, The constructed tar file. """ tf = tarfile.open(archive_path, mode='w:gz') for dpath in self.dirs: t = tf.gettarinfo(dpath) if os.path.islink(dpath): t.type = tarfile.SYMTYPE t.linkname = os.readlink(dpath) elif os.path.isdir(dpath): t.type = tarfile.DIRTYPE else: log.debug( 'Adding [%s] as dir; os.path says is neither a dir nor a link.', dpath) t.type = tarfile.DIRTYPE t.mode = os.stat(dpath).st_mode tf.addfile(_ResetOwnership(t)) log.debug('Added dir [%s]', dpath) for path in self.files: tf.add(path, filter=_ResetOwnership) log.debug('Added [%s]', path) return tf def _MakeZipFile(self, archive_path): zip_file = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED) try: for dpath in self.dirs: zip_file.write(dpath) for path in self.files: zip_file.write(path) finally: zip_file.close() def CopyArchiveToGCS( self, storage_client, gcs_object, ignore_file=None, hide_logs=False ): """Copy an archive of the snapshot to GCS. Args: storage_client: storage_api.StorageClient, The storage client to use for uploading. gcs_object: storage.objects Resource, The GCS object to write. ignore_file: Override .gcloudignore file to specify skip files. hide_logs: boolean, not print the status message if the flag is true. Returns: storage_v1_messages.Object, The written GCS object. """ with metrics.RecordDuration(metric_names.UPLOAD_SOURCE): with files.ChDir(self.src_dir): with files.TemporaryDirectory() as tmp: if gcs_object.Name().endswith('.zip'): archive_path = os.path.join(tmp, 'file.zip') self._MakeZipFile(archive_path) else: archive_path = os.path.join(tmp, 'file.tgz') tf = self._MakeTarball(archive_path) tf.close() ignore_file_path = os.path.join( self.src_dir, ignore_file or gcloudignore.IGNORE_FILE_NAME) if self.any_files_ignored: if os.path.exists(ignore_file_path): log.info('Using ignore file [{}]'.format(ignore_file_path)) elif not hide_logs: log.status.Print( _IGNORED_FILE_MESSAGE.format(log_file=log.GetLogFilePath())) if not hide_logs: file_type = ( 'zipfile' if gcs_object.Name().endswith('.zip') else 'tarball' ) log.status.write( 'Uploading {file_type} of [{src_dir}] to ' '[gs://{bucket}/{object}]\n'.format( file_type=file_type, src_dir=self.src_dir, bucket=gcs_object.bucket, object=gcs_object.object, ), ) return storage_client.CopyFileToGCS(archive_path, gcs_object)