245 lines
7.9 KiB
Python
245 lines
7.9 KiB
Python
# -*- coding: utf-8 -*- #
|
|
# Copyright 2021 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Common utilities to operate with Docker."""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import unicode_literals
|
|
|
|
import collections
|
|
import datetime
|
|
import re
|
|
from googlecloudsdk.command_lib.ai import errors
|
|
from googlecloudsdk.command_lib.ai.custom_jobs import local_util
|
|
from googlecloudsdk.core import log
|
|
|
|
_MAX_REPOSITORY_LENGTH = 255
|
|
_MAX_TAG_LENGTH = 128
|
|
|
|
_AUTONAME_PREFIX = "cloudai-autogenerated"
|
|
_DEFAULT_IMAGE_NAME = "unnamed"
|
|
_DEFAULT_REPO_REGION = "us"
|
|
|
|
Package = collections.namedtuple("Package",
|
|
["script", "package_path", "python_module"])
|
|
Image = collections.namedtuple("Image",
|
|
["name", "default_home", "default_workdir"])
|
|
|
|
|
|
def _ParseRepositoryTag(image_name):
|
|
"""Parses out the repository and tag from a Docker image name.
|
|
|
|
Args:
|
|
image_name: (str) The full name of an image, expected to be in a format of
|
|
"repository[:tag]"
|
|
|
|
Returns:
|
|
A (repository, tag) tuple representing the parsed result.
|
|
None repository means the image name is invalid; tag may be None if it isn't
|
|
present in the given image name.
|
|
"""
|
|
|
|
if image_name.count(":") > 2:
|
|
return None, None
|
|
|
|
parts = image_name.rsplit(":", 1)
|
|
if len(parts) == 2 and "/" not in parts[1]:
|
|
return tuple(parts)
|
|
|
|
return image_name, None
|
|
|
|
|
|
def _ParseRepositoryHost(repository_name):
|
|
"""Parses a repository to an optional hostname and a list of path compoentes.
|
|
|
|
Args:
|
|
repository_name: (str) A name made up of slash-separated path name
|
|
components, optionally prefixed by a registry hostname.
|
|
|
|
Returns:
|
|
A (hostname, components) tuple representing the parsed result.
|
|
The hostname will be None if it isn't present; the components is a list of
|
|
each slash-separated part in the given repository name.
|
|
"""
|
|
|
|
components = repository_name.split("/")
|
|
|
|
if len(components) == 1:
|
|
return None, components
|
|
|
|
if "." in components[0] or ":" in components[0]:
|
|
# components[0] is regarded as a hostname
|
|
return components[0], components[1:]
|
|
|
|
return None, components
|
|
|
|
|
|
def _ParseHostPort(host):
|
|
"""Parses a registry hostname to a list of components and an optional port.
|
|
|
|
Args:
|
|
host: (str) The registry hostname supposed to comply with standard DNS
|
|
rules, optionally be followed by a port number in the format like ":8080".
|
|
|
|
Returns:
|
|
A (hostcomponents, port) tuple representing the parsed result.
|
|
The hostcomponents contains each dot-seperated component in the given
|
|
hostname; port may be None if it isn't present.
|
|
"""
|
|
|
|
parts = host.rsplit(":", 1)
|
|
hostcomponents = parts[0].split(".")
|
|
port = parts[1] if len(parts) == 2 else None
|
|
|
|
return hostcomponents, port
|
|
|
|
|
|
def ValidateRepositoryAndTag(image_name):
|
|
r"""Validate the given image name is a valid repository/tag reference.
|
|
|
|
As explained in
|
|
https://docs.docker.com/engine/reference/commandline/tag/#extended-description,
|
|
a valid repository/tag reference should following the below pattern:
|
|
|
|
reference := name [ ":" tag ]
|
|
name := [hostname '/'] component ['/' component]*
|
|
hostname := hostcomponent ['.' hostcomponent]* [':' port-number]
|
|
hostcomponent := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/
|
|
port-number := /[0-9]+/
|
|
component := alpha-numeric [separator alpha-numeric]*
|
|
alpha-numeric := /[a-z0-9]+/
|
|
separator := /[_.]|__|[-]*/
|
|
|
|
tag := /[\w][\w.-]{0,127}/
|
|
|
|
Args:
|
|
image_name: (str) Full name of a Docker image.
|
|
|
|
Raises:
|
|
ValueError if the image name is not valid.
|
|
"""
|
|
repository, tag = _ParseRepositoryTag(image_name)
|
|
|
|
if repository is None:
|
|
raise ValueError("Unable to parse repository and tag.")
|
|
|
|
if len(repository) > _MAX_REPOSITORY_LENGTH:
|
|
raise ValueError(
|
|
"Repository name must not be more than {} characters.".format(
|
|
_MAX_REPOSITORY_LENGTH))
|
|
|
|
hostname, path_components = _ParseRepositoryHost(repository)
|
|
|
|
if hostname:
|
|
hostcomponents, port = _ParseHostPort(hostname)
|
|
|
|
hostcomponent_regex = r"^(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])$"
|
|
for hostcomponent in hostcomponents:
|
|
if re.match(hostcomponent_regex, hostcomponent) is None:
|
|
raise ValueError(
|
|
"Invalid hostname/port \"{}\" in repository name.".format(hostname))
|
|
|
|
port_regex = r"^[0-9]+$"
|
|
if port and re.match(port_regex, port) is None:
|
|
raise ValueError(
|
|
"Invalid hostname/port \"{}\" in repository name.".format(hostname))
|
|
|
|
for component in path_components:
|
|
if not component:
|
|
raise ValueError("Empty path component in repository name.")
|
|
|
|
component_regex = r"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*$"
|
|
if re.match(component_regex, component) is None:
|
|
raise ValueError(
|
|
"Invalid path component \"{}\" in repository name.".format(component))
|
|
|
|
if tag:
|
|
if len(tag) > _MAX_TAG_LENGTH:
|
|
raise ValueError("Tag name must not be more than {} characters.".format(
|
|
_MAX_TAG_LENGTH))
|
|
|
|
tag_regex = r"^[\w][\w.-]{0,127}$"
|
|
if re.match(tag_regex, tag) is None:
|
|
raise ValueError("Invalid tag.")
|
|
|
|
|
|
def GenerateImageName(base_name=None, project=None, region=None, is_gcr=False):
|
|
"""Generate a name for the Docker image built by AI platform gcloud."""
|
|
sanitized_name = _SanitizeRepositoryName(base_name or _DEFAULT_IMAGE_NAME)
|
|
|
|
# Use the current timestamp as the tag.
|
|
tag = datetime.datetime.now().strftime("%Y%m%d.%H.%M.%S.%f")
|
|
image_name = "{}/{}:{}".format(_AUTONAME_PREFIX, sanitized_name, tag)
|
|
|
|
if project:
|
|
if is_gcr:
|
|
repository = "gcr.io"
|
|
else:
|
|
region_prefix = region or _DEFAULT_REPO_REGION
|
|
repository = "{}-docker.pkg.dev".format(region_prefix)
|
|
return "{}/{}/{}".format(repository, project.replace(":", "/"), image_name)
|
|
return image_name
|
|
|
|
|
|
def _SanitizeRepositoryName(name):
|
|
"""Sanitizes the given name to make it valid as an image repository.
|
|
|
|
As explained in
|
|
https://docs.docker.com/engine/reference/commandline/tag/#extended-description,
|
|
Valid name may contain only lowercase letters, digits and separators.
|
|
A separator is defined as a period, one or two underscores, or one or more
|
|
dashes. A name component may not start or end with a separator.
|
|
|
|
This method will replace the illegal characters in the given name and strip
|
|
starting and ending separator characters.
|
|
|
|
Args:
|
|
name: str, the name to sanitize.
|
|
|
|
Returns:
|
|
A sanitized name.
|
|
"""
|
|
return re.sub("[._][._]+|[^a-z0-9._-]+", ".", name.lower()).strip("._-")
|
|
|
|
|
|
def ExecuteDockerCommand(command):
|
|
"""Executes Docker CLI commands in subprocess.
|
|
|
|
Just calls local_util.ExecuteCommand(cmd,...) and raises error for non-zero
|
|
exit code.
|
|
|
|
Args:
|
|
command: (List[str]) Strings to send in as the command.
|
|
|
|
Raises:
|
|
ValueError: The input command is not a docker command.
|
|
DockerError: An error occurred when executing the given docker command.
|
|
"""
|
|
|
|
command_str = " ".join(command)
|
|
if not command_str.startswith("docker"):
|
|
raise ValueError("`{}` is not a Docker command".format("docker"))
|
|
|
|
log.info("Running command: {}".format(command_str))
|
|
|
|
return_code = local_util.ExecuteCommand(command)
|
|
if return_code != 0:
|
|
error_msg = """
|
|
Docker failed with error code {code}.
|
|
Command: {cmd}
|
|
""".format(
|
|
code=return_code, cmd=command_str)
|
|
raise errors.DockerError(error_msg, command, return_code)
|