328 lines
11 KiB
Python
328 lines
11 KiB
Python
# Copyright 2016 Google LLC. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Collection of classes for converting and transforming an input dictionary.
|
|
|
|
Conversions are defined statically using subclasses of SchemaField (Message,
|
|
Value, RepeatedField) which transform a source dictionary input to the target
|
|
schema. The source dictionary is expected to be parsed from a JSON
|
|
representation.
|
|
|
|
Only fields listed in the schema will be converted (i.e. an allowlist).
|
|
A SchemaField is a recursive structure and employs the visitor pattern to
|
|
convert an input structure.
|
|
|
|
# Schema to use for transformation
|
|
SAMPLE_SCHEMA = Message(
|
|
foo=Value(target_name='bar'),
|
|
list_of_things=RepeatedField(target_name='bar_list_of_things',
|
|
element=Value()))
|
|
|
|
# Input dictionary:
|
|
input_dict = {
|
|
'foo': '1234',
|
|
'list_of_things': [1, 4, 5],
|
|
'some_other_field': "hello"
|
|
}
|
|
|
|
# To convert:
|
|
result = SAMPLE_SCHEMA.ConvertValue(input_dict)
|
|
|
|
# The resulting dictionary will be:
|
|
{
|
|
'bar': '1234',
|
|
'bar_list_of_things': [1, 4, 5]
|
|
}
|
|
|
|
Note that both fields were renamed according to the rules in the schema. Fields
|
|
not listed in the schema will not be copied. In this example, "some_other_field"
|
|
was not copied.
|
|
|
|
If further transformation is required on the value itself, a converter can be
|
|
specified, which is simply a function which takes an input value and transforms
|
|
it according to whatever logic it wants.
|
|
|
|
For example, to convert a string value to an integer value, one could construct
|
|
a schema as follows:
|
|
CONVERTER_SCHEMA = Message(
|
|
foo=Value(target_name='bar', converter=int))
|
|
|
|
Using the above input dictionary, the result would be:
|
|
{
|
|
'bar': 1234
|
|
}
|
|
"""
|
|
|
|
from __future__ import absolute_import
|
|
|
|
import logging
|
|
|
|
from googlecloudsdk.appengine.admin.tools.conversion import converters
|
|
|
|
# TODO(user) Better error handling patterns.
|
|
|
|
|
|
def UnderscoreToLowerCamelCase(text):
|
|
"""Convert underscores to lower camel case (e.g. 'foo_bar' --> 'fooBar')."""
|
|
parts = text.lower().split('_')
|
|
return parts[0] + ''.join(part.capitalize() for part in parts[1:])
|
|
|
|
|
|
def ValidateType(source_value, expected_type):
|
|
if not isinstance(source_value, expected_type):
|
|
raise ValueError(
|
|
'Expected a %s, but got %s for value %s' % (expected_type,
|
|
type(source_value),
|
|
source_value))
|
|
|
|
|
|
def ValidateNotType(source_value, non_expected_type):
|
|
if isinstance(source_value, non_expected_type):
|
|
raise ValueError(
|
|
'Did not expect %s for value %s' % (non_expected_type, source_value))
|
|
|
|
|
|
def MergeDictionaryValues(old_dict, new_dict):
|
|
"""Attempts to merge the given dictionaries.
|
|
|
|
Warns if a key exists with different values in both dictionaries. In this
|
|
case, the new_dict value trumps the previous value.
|
|
|
|
Args:
|
|
old_dict: Existing dictionary.
|
|
new_dict: New dictionary.
|
|
|
|
Returns:
|
|
Result of merging the two dictionaries.
|
|
|
|
Raises:
|
|
ValueError: If the keys in each dictionary are not unique.
|
|
"""
|
|
common_keys = set(old_dict) & set(new_dict)
|
|
if common_keys:
|
|
conflicting_keys = set(key for key in common_keys
|
|
if old_dict[key] != new_dict[key])
|
|
if conflicting_keys:
|
|
def FormatKey(key):
|
|
return ('\'{key}\' has conflicting values \'{old}\' and \'{new}\'. '
|
|
'Using \'{new}\'.').format(key=key,
|
|
old=old_dict[key],
|
|
new=new_dict[key])
|
|
for conflicting_key in conflicting_keys:
|
|
logging.warning(FormatKey(conflicting_key))
|
|
result = old_dict.copy()
|
|
result.update(new_dict)
|
|
return result
|
|
|
|
|
|
class SchemaField(object):
|
|
"""Transformation strategy from input dictionary to an output dictionary.
|
|
|
|
Each subclass defines a different strategy for how an input value is converted
|
|
to an output value. ConvertValue() makes a copy of the input with the proper
|
|
transformations applied. Additionally, constraints about the input structure
|
|
are validated while doing the transformation.
|
|
"""
|
|
|
|
def __init__(self, target_name=None, converter=None):
|
|
"""Constructor.
|
|
|
|
Args:
|
|
target_name: New field name to use when creating an output dictionary. If
|
|
None is specified, then the original name is used.
|
|
converter: A function which performs a transformation on the value of the
|
|
field.
|
|
"""
|
|
self.target_name = target_name
|
|
self.converter = converter
|
|
|
|
def ConvertValue(self, value):
|
|
"""Convert an input value using the given schema and converter.
|
|
|
|
This method is not meant to be overwritten. Update _VisitInternal to change
|
|
the behavior.
|
|
|
|
Args:
|
|
value: Input value.
|
|
|
|
Returns:
|
|
Output which has been transformed using the given schema for renaming and
|
|
converter, if specified.
|
|
"""
|
|
result = self._VisitInternal(value)
|
|
return self._PerformConversion(result)
|
|
|
|
def _VisitInternal(self, value):
|
|
"""Shuffles the input value using the renames specified in the schema.
|
|
|
|
Only structural changes are made (e.g. renaming keys, copying lists, etc.).
|
|
Subclasses are expected to override this.
|
|
|
|
Args:
|
|
value: Input value.
|
|
|
|
Returns:
|
|
Output which has been transformed using the given schema.
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
def _PerformConversion(self, result):
|
|
"""Transforms the result value if a converter is specified."""
|
|
return self.converter(result) if self.converter else result
|
|
|
|
|
|
class Message(SchemaField):
|
|
"""A message has a collection of fields which should be converted.
|
|
|
|
Expected input type: Dictionary
|
|
Output type: Dictionary
|
|
"""
|
|
|
|
def __init__(self, target_name=None, converter=None, **kwargs):
|
|
"""Constructor.
|
|
|
|
Args:
|
|
target_name: New field name to use when creating an output dictionary. If
|
|
None is specified, then the original name is used.
|
|
converter: A function which performs a transformation on the value of the
|
|
field.
|
|
**kwargs: Kwargs where the keys are names of the fields and values are
|
|
FieldSchemas for each child field.
|
|
|
|
Raises:
|
|
ValueError: If the message has no child fields specified.
|
|
"""
|
|
super(Message, self).__init__(target_name, converter)
|
|
self.fields = kwargs
|
|
if not self.fields:
|
|
raise ValueError('Message must contain fields')
|
|
|
|
def _VisitInternal(self, value):
|
|
"""Convert each child field and put the result in a new dictionary."""
|
|
ValidateType(value, dict)
|
|
result = {}
|
|
for source_key, field_schema in self.fields.items():
|
|
if source_key not in value:
|
|
continue
|
|
|
|
source_value = value[source_key]
|
|
target_key = field_schema.target_name or source_key
|
|
target_key = UnderscoreToLowerCamelCase(target_key)
|
|
|
|
result_value = field_schema.ConvertValue(source_value)
|
|
if target_key not in result:
|
|
result[target_key] = result_value
|
|
# Only know how to merge dicts right now.
|
|
elif isinstance(result[target_key], dict) and isinstance(result_value,
|
|
dict):
|
|
result[target_key] = MergeDictionaryValues(result[target_key],
|
|
result_value)
|
|
else:
|
|
raise ValueError('Target key "%s" already exists.' % target_key)
|
|
|
|
return result
|
|
|
|
|
|
class Value(SchemaField):
|
|
"""Represents a leaf node. Only the value itself is copied.
|
|
|
|
A primitive value corresponds to any non-string, non-dictionary value which
|
|
can be represented in JSON.
|
|
|
|
Expected input type: Primitive value type (int, string, boolean, etc.).
|
|
Output type: Same primitive value type.
|
|
"""
|
|
|
|
def _VisitInternal(self, value):
|
|
ValidateNotType(value, list)
|
|
ValidateNotType(value, dict)
|
|
return value
|
|
|
|
|
|
class Map(SchemaField):
|
|
"""Represents a leaf node where the value itself is a map.
|
|
|
|
Expected input type: Dictionary
|
|
Output type: Dictionary
|
|
"""
|
|
|
|
def __init__(self, target_name=None, converter=None,
|
|
key_converter=converters.ToJsonString,
|
|
value_converter=converters.ToJsonString):
|
|
"""Constructor.
|
|
|
|
Args:
|
|
target_name: New field name to use when creating an output dictionary. If
|
|
None is specified, then the original name is used.
|
|
converter: A function which performs a transformation on the value of the
|
|
field.
|
|
key_converter: A function which performs a transformation on the keys.
|
|
value_converter: A function which performs a transformation on the values.
|
|
"""
|
|
super(Map, self).__init__(target_name, converter)
|
|
self.key_converter = key_converter
|
|
self.value_converter = value_converter
|
|
|
|
def _VisitInternal(self, value):
|
|
ValidateType(value, dict)
|
|
result = {}
|
|
for key, dict_value in value.items():
|
|
if self.key_converter:
|
|
key = self.key_converter(key)
|
|
if self.value_converter:
|
|
dict_value = self.value_converter(dict_value)
|
|
result[key] = dict_value
|
|
|
|
return result
|
|
|
|
|
|
class RepeatedField(SchemaField):
|
|
"""Represents a list of nested elements. Each item in the list is copied.
|
|
|
|
The type of each element in the list is specified in the constructor.
|
|
|
|
Expected input type: List
|
|
Output type: List
|
|
"""
|
|
|
|
def __init__(self, target_name=None, converter=None, element=None):
|
|
"""Constructor.
|
|
|
|
Args:
|
|
target_name: New field name to use when creating an output dictionary. If
|
|
None is specified, then the original name is used.
|
|
converter: A function which performs a transformation on the value of the
|
|
field.
|
|
element: A SchemaField element defining the type of every element in the
|
|
list. The input structure is expected to be homogenous.
|
|
|
|
Raises:
|
|
ValueError: If an element has not been specified or if the element type is
|
|
incompatible with a repeated field.
|
|
"""
|
|
super(RepeatedField, self).__init__(target_name, converter)
|
|
self.element = element
|
|
|
|
if not self.element:
|
|
raise ValueError('Element required for a repeated field')
|
|
|
|
if isinstance(self.element, Map):
|
|
raise ValueError('Repeated maps are not supported')
|
|
|
|
def _VisitInternal(self, value):
|
|
ValidateType(value, list)
|
|
result = []
|
|
for item in value:
|
|
result.append(self.element.ConvertValue(item))
|
|
return result
|