feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/lib/third_party/hcl2/LICENSE

@@ -0,0 +1,19 @@
+Copyright 2017 Amplify Education, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

gcloud auth application-default login/google-cloud-sdk/lib/third_party/hcl2/__init__.py

@@ -0,0 +1,8 @@
+"""For package documentation, see README"""
+
+try:
+    from .version import version as __version__
+except ImportError:
+    __version__ = "unknown"
+
+from .api import load, loads

gcloud auth application-default login/google-cloud-sdk/lib/third_party/hcl2/__main__.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+"""
+This script recursively converts hcl2 files to json
+
+Usage:
+    hcl2tojson [-s] PATH [OUT_PATH]
+
+Options:
+    -s          Skip un-parsable files
+    PATH        The path to convert
+    OUT_PATH    The path to write files to
+    --with-meta If set add meta parameters to the output_json like __start_line__ and __end_line__
+"""
+import argparse
+import json
+import os
+import sys
+
+from lark import UnexpectedCharacters, UnexpectedToken
+
+from . import load
+from .version import __version__
+
+
+def main():
+    """The `console_scripts` entry point"""
+
+    parser = argparse.ArgumentParser(
+        description="This script recursively converts hcl2 files to json"
+    )
+    parser.add_argument(
+        "-s", dest="skip", action="store_true", help="Skip un-parsable files"
+    )
+    parser.add_argument("PATH", help="The file or directory to convert")
+    parser.add_argument(
+        "OUT_PATH",
+        nargs="?",
+        help="The path where to write files to. Optional when parsing a single file. "
+        "Output is printed to stdout if OUT_PATH is blank",
+    )
+    parser.add_argument("--version", action="version", version=__version__)
+    parser.add_argument(
+        "--with-meta",
+        action="store_true",
+        help="If set add meta parameters to the output_json like __start_line__ and __end_line__",
+    )
+
+    args = parser.parse_args()
+
+    skippable_exceptions = (UnexpectedToken, UnexpectedCharacters, UnicodeDecodeError)
+
+    if os.path.isfile(args.PATH):
+        with open(args.PATH, "r", encoding="utf-8") as in_file:
+            # pylint: disable=R1732
+            out_file = (
+                sys.stdout
+                if args.OUT_PATH is None
+                else open(args.OUT_PATH, "w", encoding="utf-8")
+            )
+            print(args.PATH, file=sys.stderr, flush=True)
+            json.dump(load(in_file, with_meta=args.with_meta), out_file)
+            if args.OUT_PATH is None:
+                out_file.write("\n")
+                out_file.close()
+    elif os.path.isdir(args.PATH):
+        processed_files = set()
+        if args.OUT_PATH is None:
+            raise RuntimeError("Positional OUT_PATH parameter shouldn't be empty")
+        if not os.path.exists(args.OUT_PATH):
+            os.mkdir(args.OUT_PATH)
+        for current_dir, _, files in os.walk(args.PATH):
+            dir_prefix = os.path.commonpath([args.PATH, current_dir])
+            relative_current_dir = os.path.relpath(current_dir, dir_prefix)
+            current_out_path = os.path.normpath(
+                os.path.join(args.OUT_PATH, relative_current_dir)
+            )
+            if not os.path.exists(current_out_path):
+                os.mkdir(current_out_path)
+            for file_name in files:
+                in_file_path = os.path.join(current_dir, file_name)
+                out_file_path = os.path.join(current_out_path, file_name)
+                out_file_path = os.path.splitext(out_file_path)[0] + ".json"
+
+                # skip any files that we already processed or generated to avoid loops and file lock errors
+                if in_file_path in processed_files or out_file_path in processed_files:
+                    continue
+
+                processed_files.add(in_file_path)
+                processed_files.add(out_file_path)
+
+                with open(in_file_path, "r", encoding="utf-8") as in_file:
+                    print(in_file_path, file=sys.stderr, flush=True)
+                    try:
+                        parsed_data = load(in_file)
+                    except skippable_exceptions:
+                        if args.skip:
+                            continue
+                        raise
+                    with open(out_file_path, "w", encoding="utf-8") as out_file:
+                        json.dump(parsed_data, out_file)
+    else:
+        raise RuntimeError("Invalid Path", args.PATH)
+
+
+if __name__ == "__main__":
+    main()

gcloud auth application-default login/google-cloud-sdk/lib/third_party/hcl2/api.py

@@ -0,0 +1,28 @@
+"""The API that will be exposed to users of this package"""
+from typing import TextIO
+
+from hcl2.parser import hcl2
+from hcl2.transformer import DictTransformer
+
+
+def load(file: TextIO, with_meta=False) -> dict:
+    """Load a HCL2 file.
+    :param file: File with hcl2 to be loaded as a dict.
+    :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
+    parameters to the output dict. Default to false.
+    """
+    return loads(file.read(), with_meta=with_meta)
+
+
+def loads(text: str, with_meta=False) -> dict:
+    """Load HCL2 from a string.
+    :param text: Text with hcl2 to be loaded as a dict.
+    :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
+    parameters to the output dict. Default to false.
+    """
+    # append new line as a workaround for https://github.com/lark-parser/lark/issues/237
+    # Lark doesn't support a EOF token so our grammar can't look for "new line or end of file"
+    # This means that all blocks must end in a new line even if the file ends
+    # Append a new line as a temporary fix
+    tree = hcl2.parse(text + "\n")
+    return DictTransformer(with_meta=with_meta).transform(tree)

gcloud auth application-default login/google-cloud-sdk/lib/third_party/hcl2/hcl2.lark

@@ -0,0 +1,74 @@
+start : body
+body : (new_line_or_comment? (attribute | block))* new_line_or_comment?
+attribute : identifier "=" expression
+block : identifier (identifier | STRING_LIT)* new_line_or_comment? "{" body "}"
+new_line_and_or_comma: new_line_or_comment | "," | "," new_line_or_comment
+new_line_or_comment: ( /\n/ | /#.*\n/ | /\/\/.*\n/ )+
+
+identifier : /[a-zA-Z_][a-zA-Z0-9_-]*/
+
+?expression : expr_term | operation | conditional
+
+conditional : expression "?" new_line_or_comment? expression new_line_or_comment? ":" new_line_or_comment? expression
+
+?operation : unary_op | binary_op
+!unary_op : ("-" | "!") expr_term
+binary_op : expression binary_term new_line_or_comment?
+!binary_operator : "==" | "!=" | "<" | ">" | "<=" | ">=" | "-" | "*" | "/" | "%" | "&&" | "||" | "+"
+binary_term : binary_operator new_line_or_comment? expression
+
+expr_term : "(" new_line_or_comment? expression new_line_or_comment? ")"
+            | float_lit
+            | int_lit
+            | STRING_LIT
+            | tuple
+            | object
+            | function_call
+            | index_expr_term
+            | get_attr_expr_term
+            | identifier
+            | heredoc_template
+            | heredoc_template_trim
+            | attr_splat_expr_term
+            | full_splat_expr_term
+            | for_tuple_expr
+            | for_object_expr
+
+
+STRING_LIT : "\"" (STRING_CHARS | INTERPOLATION)* "\""
+STRING_CHARS : /(?:(?!\${)([^"\\]|\\.))+/+ // any character except '"" unless inside a interpolation string
+NESTED_INTERPOLATION : "${" /[^}]+/ "}"
+INTERPOLATION : "${" (/(?:(?!\${)([^}]))+/ | NESTED_INTERPOLATION)+ "}"
+
+int_lit : DECIMAL+
+!float_lit: DECIMAL+ "." DECIMAL+ (EXP_MARK DECIMAL+)?
+            | DECIMAL+ ("." DECIMAL+)? EXP_MARK DECIMAL+
+DECIMAL : "0".."9"
+EXP_MARK : ("e" | "E") ("+" | "-")?
+
+tuple : "[" (new_line_or_comment* expression new_line_or_comment* ",")* (new_line_or_comment* expression)? new_line_or_comment* "]"
+object : "{" new_line_or_comment? (object_elem (new_line_and_or_comma object_elem )* new_line_and_or_comma?)? "}"
+object_elem : (identifier | expression) ("=" | ":") expression
+
+heredoc_template : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:.|\n)*?(?P=heredoc)/
+heredoc_template_trim : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]+)\n(?:.|\n)*?(?P=heredoc_trim)/
+
+function_call : identifier "(" new_line_or_comment? arguments? new_line_or_comment? ")"
+arguments : (expression (new_line_or_comment* "," new_line_or_comment*  expression)* ("," | "...")? new_line_or_comment*)
+
+index_expr_term : expr_term index
+get_attr_expr_term : expr_term get_attr
+attr_splat_expr_term : expr_term attr_splat
+full_splat_expr_term : expr_term full_splat
+index : "[" new_line_or_comment? expression new_line_or_comment? "]" | "." DECIMAL+
+get_attr : "." identifier
+attr_splat : ".*" get_attr*
+full_splat : "[*]" (get_attr | index)*
+
+!for_tuple_expr : "[" new_line_or_comment? for_intro new_line_or_comment? expression new_line_or_comment? for_cond? new_line_or_comment? "]"
+!for_object_expr : "{" new_line_or_comment? for_intro new_line_or_comment? expression "=>" new_line_or_comment? expression "..."? new_line_or_comment? for_cond? new_line_or_comment? "}"
+!for_intro : "for" new_line_or_comment? identifier ("," identifier new_line_or_comment?)? new_line_or_comment? "in" new_line_or_comment? expression new_line_or_comment? ":" new_line_or_comment?
+!for_cond : "if" new_line_or_comment? expression
+
+%ignore /[ \t]+/
+%ignore /\/\*(.|\n)*?(\*\/)/

gcloud auth application-default login/google-cloud-sdk/lib/third_party/hcl2/parser.py

@@ -0,0 +1,16 @@
+"""A parser for HCL2 implemented using the Lark parser"""
+from pathlib import Path
+
+from lark import Lark
+
+
+PARSER_FILE = Path(__file__).absolute().resolve().parent / ".lark_cache.bin"
+
+
+hcl2 = Lark.open(
+    "hcl2.lark",
+    parser="lalr",
+    cache=str(PARSER_FILE),  # Disable/Delete file to effect changes to the grammar
+    rel_to=__file__,
+    propagate_positions=True,
+)

gcloud auth application-default login/google-cloud-sdk/lib/third_party/hcl2/transformer.py

@@ -0,0 +1,289 @@
+"""A Lark Transformer for transforming a Lark parse tree into a Python dict"""
+import re
+import sys
+from collections import namedtuple
+from typing import List, Dict, Any
+
+from lark.tree import Meta
+from lark.visitors import Transformer, Discard, _DiscardType, v_args
+
+
+HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S)
+HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S)
+
+
+START_LINE = "__start_line__"
+END_LINE = "__end_line__"
+
+
+Attribute = namedtuple("Attribute", ("key", "value"))
+
+
+# pylint: disable=missing-function-docstring,unused-argument
+class DictTransformer(Transformer):
+    """Takes a syntax tree generated by the parser and
+    transforms it to a dict.
+    """
+
+    with_meta: bool
+
+    def __init__(self, with_meta: bool = False):
+        """
+        :param with_meta: If set to true then adds `__start_line__` and `__end_line__`
+        parameters to the output dict. Default to false.
+        """
+        self.with_meta = with_meta
+        super().__init__()
+
+    def float_lit(self, args: List) -> float:
+        return float("".join([str(arg) for arg in args]))
+
+    def int_lit(self, args: List) -> int:
+        return int("".join([str(arg) for arg in args]))
+
+    def expr_term(self, args: List) -> Any:
+        args = self.strip_new_line_tokens(args)
+
+        #
+        if args[0] == "true":
+            return True
+        if args[0] == "false":
+            return False
+        if args[0] == "null":
+            return None
+
+        # if the expression starts with a paren then unwrap it
+        if args[0] == "(":
+            return args[1]
+        # otherwise return the value itself
+        return args[0]
+
+    def index_expr_term(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        return f"{args[0]}{args[1]}"
+
+    def index(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        return f"[{args[0]}]"
+
+    def get_attr_expr_term(self, args: List) -> str:
+        return f"{args[0]}{args[1]}"
+
+    def get_attr(self, args: List) -> str:
+        return f".{args[0]}"
+
+    def attr_splat_expr_term(self, args: List) -> str:
+        return f"{args[0]}{args[1]}"
+
+    def attr_splat(self, args: List) -> str:
+        args_str = "".join(str(arg) for arg in args)
+        return f".*{args_str}"
+
+    def full_splat_expr_term(self, args: List) -> str:
+        return f"{args[0]}{args[1]}"
+
+    def full_splat(self, args: List) -> str:
+        args_str = "".join(str(arg) for arg in args)
+        return f"[*]{args_str}"
+
+    def tuple(self, args: List) -> List:
+        return [self.to_string_dollar(arg) for arg in self.strip_new_line_tokens(args)]
+
+    def object_elem(self, args: List) -> Dict:
+        # This returns a dict with a single key/value pair to make it easier to merge these
+        # into a bigger dict that is returned by the "object" function
+        key = self.strip_quotes(args[0])
+        value = self.to_string_dollar(args[1])
+
+        return {key: value}
+
+    def object(self, args: List) -> Dict:
+        args = self.strip_new_line_tokens(args)
+        result: Dict[str, Any] = {}
+        for arg in args:
+            result.update(arg)
+        return result
+
+    def function_call(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        args_str = ""
+        if len(args) > 1:
+            args_str = ", ".join([str(arg) for arg in args[1] if arg is not Discard])
+        return f"{args[0]}({args_str})"
+
+    def arguments(self, args: List) -> List:
+        return args
+
+    def new_line_and_or_comma(self, args: List) -> _DiscardType:
+        return Discard
+
+    @v_args(meta=True)
+    def block(self, meta: Meta, args: List) -> Dict:
+        *block_labels, block_body = args
+        result: Dict[str, Any] = block_body
+        if self.with_meta:
+            result.update(
+                {
+                    START_LINE: meta.line,
+                    END_LINE: meta.end_line,
+                }
+            )
+
+        # create nested dict. i.e. {label1: {label2: {labelN: result}}}
+        for label in reversed(block_labels):
+            label_str = self.strip_quotes(label)
+            result = {label_str: result}
+
+        return result
+
+    def attribute(self, args: List) -> Attribute:
+        key = str(args[0])
+        if key.startswith('"') and key.endswith('"'):
+            key = key[1:-1]
+        value = self.to_string_dollar(args[1])
+        return Attribute(key, value)
+
+    def conditional(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        return f"{args[0]} ? {args[1]} : {args[2]}"
+
+    def binary_op(self, args: List) -> str:
+        return " ".join([str(arg) for arg in args])
+
+    def unary_op(self, args: List) -> str:
+        return "".join([str(arg) for arg in args])
+
+    def binary_term(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        return " ".join([str(arg) for arg in args])
+
+    def body(self, args: List) -> Dict[str, List]:
+        # See https://github.com/hashicorp/hcl/blob/main/hclsyntax/spec.md#bodies
+        # ---
+        # A body is a collection of associated attributes and blocks.
+        #
+        # An attribute definition assigns a value to a particular attribute
+        # name within a body. Each distinct attribute name may be defined no
+        # more than once within a single body.
+        #
+        # A block creates a child body that is annotated with a block type and
+        # zero or more block labels. Blocks create a structural hierarchy which
+        # can be interpreted by the calling application.
+        # ---
+        #
+        # There can be more than one child body with the same block type and
+        # labels. This means that all blocks (even when there is only one)
+        # should be transformed into lists of blocks.
+        args = self.strip_new_line_tokens(args)
+        attributes = set()
+        result: Dict[str, Any] = {}
+        for arg in args:
+            if isinstance(arg, Attribute):
+                if arg.key in result:
+                    raise RuntimeError(f"{arg.key} already defined")
+                result[arg.key] = arg.value
+                attributes.add(arg.key)
+            else:
+                # This is a block.
+                for key, value in arg.items():
+                    key = str(key)
+                    if key in result:
+                        if key in attributes:
+                            raise RuntimeError(f"{key} already defined")
+                        result[key].append(value)
+                    else:
+                        result[key] = [value]
+
+        return result
+
+    def start(self, args: List) -> Dict:
+        args = self.strip_new_line_tokens(args)
+        return args[0]
+
+    def binary_operator(self, args: List) -> str:
+        return str(args[0])
+
+    def heredoc_template(self, args: List) -> str:
+        match = HEREDOC_PATTERN.match(str(args[0]))
+        if not match:
+            raise RuntimeError(f"Invalid Heredoc token: {args[0]}")
+
+        trim_chars = "\n\t "
+        return f'"{match.group(2).rstrip(trim_chars)}"'
+
+    def heredoc_template_trim(self, args: List) -> str:
+        # See https://github.com/hashicorp/hcl2/blob/master/hcl/hclsyntax/spec.md#template-expressions
+        # This is a special version of heredocs that are declared with "<<-"
+        # This will calculate the minimum number of leading spaces in each line of a heredoc
+        # and then remove that number of spaces from each line
+        match = HEREDOC_TRIM_PATTERN.match(str(args[0]))
+        if not match:
+            raise RuntimeError(f"Invalid Heredoc token: {args[0]}")
+
+        trim_chars = "\n\t "
+        text = match.group(2).rstrip(trim_chars)
+        lines = text.split("\n")
+
+        # calculate the min number of leading spaces in each line
+        min_spaces = sys.maxsize
+        for line in lines:
+            leading_spaces = len(line) - len(line.lstrip(" "))
+            min_spaces = min(min_spaces, leading_spaces)
+
+        # trim off that number of leading spaces from each line
+        lines = [line[min_spaces:] for line in lines]
+
+        return '"%s"' % "\n".join(lines)
+
+    def new_line_or_comment(self, args: List) -> _DiscardType:
+        return Discard
+
+    def for_tuple_expr(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        for_expr = " ".join([str(arg) for arg in args[1:-1]])
+        return f"[{for_expr}]"
+
+    def for_intro(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        return " ".join([str(arg) for arg in args])
+
+    def for_cond(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        return " ".join([str(arg) for arg in args])
+
+    def for_object_expr(self, args: List) -> str:
+        args = self.strip_new_line_tokens(args)
+        for_expr = " ".join([str(arg) for arg in args[1:-1]])
+        # doubled curly braces stands for inlining the braces
+        # and the third pair of braces is for the interpolation
+        # e.g. f"{2 + 2} {{2 + 2}}" == "4 {2 + 2}"
+        return f"{{{for_expr}}}"
+
+    def strip_new_line_tokens(self, args: List) -> List:
+        """
+        Remove new line and Discard tokens.
+        The parser will sometimes include these in the tree so we need to strip them out here
+        """
+        return [arg for arg in args if arg != "\n" and arg is not Discard]
+
+    def to_string_dollar(self, value: Any) -> Any:
+        """Wrap a string in ${ and }"""
+        if isinstance(value, str):
+            if value.startswith('"') and value.endswith('"'):
+                return str(value)[1:-1]
+            return f"${{{value}}}"
+        return value
+
+    def strip_quotes(self, value: Any) -> Any:
+        """Remove quote characters from the start and end of a string"""
+        if isinstance(value, str):
+            if value.startswith('"') and value.endswith('"'):
+                return str(value)[1:-1]
+        return value
+
+    def identifier(self, value: Any) -> Any:
+        # Making identifier a token by capitalizing it to IDENTIFIER
+        # seems to return a token object instead of the str
+        # So treat it like a regular rule
+        # In this case we just convert the whole thing to a string
+        return str(value[0])

gcloud auth application-default login/google-cloud-sdk/lib/third_party/hcl2/version.py

@@ -0,0 +1,4 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+__version__ = version = '4.3.2'
+__version_tuple__ = version_tuple = (4, 3, 2)