feat: Add new gcloud commands, API clients, and third-party libraries across various services.

gcloud auth application-default login/google-cloud-sdk/platform/gsutil/third_party/httplib2/script/compile-py3-openssl11.sh

@@ -0,0 +1,60 @@
+#!not for running standalone, see .github/workflows/test.yaml
+
+cache_dir=$HOME/.cache
+install_dir=$cache_dir/py3-openssl11
+python_version="3.7.3"
+openssl_version="1.1.1f"
+cpucount=$(nproc --all)
+export PYTHONDONTWRITEBYTECODE=1
+
+#rm -rf $cache_dir/* # uncomment to rebuild
+
+if [[ ! -x "$install_dir/bin/python" ]] || [[ $($install_dir/bin/python -V) != "Python $python_version" ]] ; then
+  (
+  mkdir -p /tmp/source
+  cd /tmp/source
+  # Compile OpenSSL
+  curl -fLOsS "https://www.openssl.org/source/openssl-$openssl_version.tar.gz"
+  echo "Extracting OpenSSL..."
+  tar xf openssl-$openssl_version.tar.gz
+  cd ./openssl-$openssl_version
+  echo "Compiling OpenSSL $openssl_version..."
+  ./config shared --prefix=$install_dir
+  echo "Running make for OpenSSL..."
+  make -j$cpucount -s
+  echo "Running make install for OpenSSL..."
+  make install_sw >/dev/null
+  export LD_LIBRARY_PATH=$install_dir/lib
+
+  cd /tmp/source
+  sudo apt install -qq --yes libffi-dev
+  # Compile latest Python
+  curl -fLOsS "https://www.python.org/ftp/python/$python_version/Python-$python_version.tar.xz"
+  echo "Extracting Python..."
+  tar xf Python-$python_version.tar.xz
+  cd ./Python-$python_version
+  echo "Compiling Python $python_version..."
+  # Note we are purposefully NOT using optimization flags as they increase compile time 10x
+  conf_flags="--with-openssl=$install_dir --prefix=$install_dir --with-ensurepip=upgrade"
+  CFLAGS=-O1 ./configure $conf_flags > /dev/null
+  make -j$cpucount -s
+  echo "Installing Python..."
+  make altinstall bininstall >/dev/null
+  ln -fs pip3.7 $install_dir/bin/pip3
+  ln -fs pip3 $install_dir/bin/pip
+  ln -fs python3 $install_dir/bin/python
+  # care for CI cache size
+  find $install_dir -type d -name __pycache__ -print0 |xargs -0 rm -rf
+  )
+fi
+
+export LD_LIBRARY_PATH=$install_dir/lib
+export PATH=$install_dir/bin:$PATH
+if [[ $(python -V) != "Python $python_version" ]] ; then
+  echo "Required Python version was not installed into PATH" >&2
+  exit 1
+fi
+if [[ $(python -c 'import ssl; print(ssl.OPENSSL_VERSION)') != OpenSSL\ ${openssl_version}* ]] ; then
+  echo "Required OpenSSL version was not installed into Python" >&2
+  exit 1
+fi

gcloud auth application-default login/google-cloud-sdk/platform/gsutil/third_party/httplib2/script/generate-tls

@@ -0,0 +1,61 @@
+#!/bin/bash
+set -eu
+
+target_dir="${1:-.}"
+days=7300
+rsa_bits=2048
+org="httplib2-test"
+server_cn="localhost"
+subj_prefix="/C=ZZ/ST=./L=./O=$org/OU=."
+
+main() {
+	cd "$target_dir"
+	gen
+	check
+}
+
+check() {
+	echo "- check keys" >&2
+	openssl rsa -in ca.key -check -noout
+	openssl rsa -in client.key -check -noout
+	openssl rsa -in client_encrypted.key -check -noout -passin pass:12345
+	openssl rsa -in server.key -check -noout
+
+	echo "- check certs" >&2
+	for f in *.pem ; do
+		openssl x509 -in "$f" -checkend 3600 -noout
+	done
+}
+
+gen() {
+	echo "- generate keys, if absent" >&2
+	[[ -f ca.key ]] || openssl genrsa -out ca.key $rsa_bits
+	[[ -f client.key ]] || openssl genrsa -out client.key $rsa_bits
+	[[ -f client_encrypted.key ]] || openssl rsa -in client.key -out client_encrypted.key -aes128 -passout pass:12345
+	[[ -f server.key ]] || openssl genrsa -out server.key $rsa_bits
+
+	echo "- generate CA" >&2
+	openssl req -batch -new -nodes -x509 -days $days -subj "$subj_prefix/CN=$org-CA" -key ca.key -out ca.pem
+	openssl req -batch -new -nodes -x509 -days $days -subj "$subj_prefix/CN=$org-CA-unused" -key ca.key -out ca_unused.pem
+
+	echo "- generate client cert" >&2
+	openssl req -batch -new -nodes -out tmp.csr -key client.key -subj "$subj_prefix/CN=$org-client"
+	openssl x509 -req -in tmp.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out client.crt -days $days -serial -fingerprint
+	cat client.crt client.key >client.pem
+	cat client.crt ca.pem client.key >client_chain.pem
+
+	echo "- generate encrypted client cert" >&2
+	openssl req -batch -new -nodes -out tmp.csr -key client_encrypted.key -passin pass:12345 -subj "$subj_prefix/CN=$org-client-enc"
+	openssl x509 -req -in tmp.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out client_encrypted.crt -days $days -serial -fingerprint
+	cat client_encrypted.crt client_encrypted.key >client_encrypted.pem
+
+	echo "- generate server cert" >&2
+	openssl req -batch -new -nodes -out tmp.csr -key server.key -subj "$subj_prefix/CN=$server_cn"
+	openssl x509 -req -in tmp.csr -CA ca.pem -CAkey ca.key -CAcreateserial -out server.crt -days $days -serial -fingerprint
+	cat server.crt server.key >server.pem
+	cat server.crt ca.pem server.key >server_chain.pem
+
+	rm tmp.csr
+}
+
+main

gcloud auth application-default login/google-cloud-sdk/platform/gsutil/third_party/httplib2/script/release

@@ -0,0 +1,244 @@
+#!/bin/bash
+set -eu
+cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+
+version=
+version_next=
+
+main() {
+	local opt_auto=0
+	while [[ $# -gt 0 ]] ; do
+		case $1 in
+		-auto)
+			opt_auto=1
+			;;
+		*)
+			echo "Usage: $0 [-auto]" >&2
+			exit 0
+			;;
+		esac
+		shift
+	done
+	if [[ "$opt_auto" -eq 1 ]] ; then
+		auto_prepare_release
+	else
+		interactive
+	fi
+}
+
+auto_prepare_release() {
+	echo 'script/release: auto mode for CI, will check or modify version based on tag' >&2
+
+	assert_tree_clean
+
+	local is_tag=0
+	local version_tag=
+	if version_tag=$(git describe --candidates=0 --tags HEAD 2>/dev/null) ; then
+		is_tag=1
+		version_tag=${version_tag##v}
+		version_check "$version_tag"
+	fi
+
+	local last_tag=
+	local version_replace=
+	if [[ "$is_tag" -eq 0 ]] ; then
+		last_tag=$(git tag --sort=-version:refname |head -n1)
+		last_tag=${last_tag##v}
+		version_replace="${last_tag}.post$(date -u +%y%m%d%H%M)"
+		update_version "setup.py" "s/VERSION =.+/VERSION = \"$version_replace\"/"
+		update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_replace\"/"
+		update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_replace\"/"
+		version_check "$version_replace"
+	fi
+}
+
+interactive() {
+	echo 'script/release: interactive mode for creating new tagged releases with human assistance' >&2
+
+	local branch="${1-$(git symbolic-ref --short HEAD)}"
+	version=$(python -Es -c "$(egrep '^__version__' python3/httplib2/__init__.py);print(__version__)")
+	printf "\nbranch: %s httplib2.__version__: '%s'\n" $branch $version >&2
+
+	if [[ "$branch" != "master" ]] ; then
+		echo "Must be on master" >&2
+		exit 1
+	fi
+	assert_tree_clean
+
+	last_commit_message=$(git show --format="%s" --no-patch HEAD)
+	expect_commit_message="v$version release"
+	if [[ "$last_commit_message" != "$expect_commit_message" ]] ; then
+		printf "Last commit message: '%s' expected: '%s'\n" "$last_commit_message" "$expect_commit_message" >&2
+		if confirm "Create release commit? [yN] " ; then
+			create_commit
+		elif ! confirm "Continue without proper release commit? [yN] " ; then
+			exit 1
+		fi
+	fi
+	confirm "Continue? [yN] " || exit 1
+
+	echo "Creating tag v$version" >&2
+	if ! git tag "v$version" ; then
+		echo "git tag failed " >&2
+		confirm "Continue still? [yN] " || exit 1
+	fi
+
+	echo "Building package" >&2
+	find . -name '*.pyc' -o -name '*.pyo' -o -name '*.orig' -delete
+	rm -rf python{2,3}/.cache
+	rm -rf build dist
+	local venv=./venv-release
+	if [[ ! -d "$venv" ]] ; then
+		virtualenv $venv
+		$venv/bin/pip install -U check-manifest pip 'setuptools>=43.0' wheel twine
+	fi
+	$venv/bin/python setup.py clean --all
+	$venv/bin/python setup.py sdist bdist_wheel
+	$venv/bin/check-manifest || echo "FIXME check-manifest" >&2
+
+	if confirm "Upload to PyPI? Use in special situation, normally CI/CD will upload to PyPI. [yN] " ; then
+		$venv/bin/twine upload dist/* || exit 1
+	fi
+
+	git push --tags
+}
+
+create_commit() {
+	echo "" >&2
+	echo "Plan:" >&2
+	echo "1. bump version" >&2
+	echo "2. update CHANGELOG" >&2
+	echo "3. commit" >&2
+	echo "4. run bin/release again" >&2
+	echo "" >&2
+
+	bump_version
+	edit_news
+
+	git diff
+	confirm "Ready to commit? [Yn] " || exit 1
+	git commit -a -m "v$version_next release"
+
+	echo "Re-exec $0 to continue" >&2
+	exec $0
+}
+
+bump_version() {
+	local current=$version
+	echo "Current version: '$current'" >&2
+	echo -n "Enter next version (empty to abort): " >&2
+	read version_next
+	if [[ -z "$version_next" ]] ; then
+		exit 1
+	fi
+	echo "Next version:    '$version_next'" >&2
+
+	update_version "python3/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_next\"/"
+	update_version "python2/httplib2/__init__.py" "s/__version__ =.+/__version__ = \"$version_next\"/"
+	update_version "setup.py" "s/VERSION =.+/VERSION = \"$version_next\"/"
+
+	confirm "Confirm changes? [yN] " || exit 1
+}
+
+update_version() {
+	local path="$1"
+	local sed_expr="$2"
+		# sed -E --in-place='' -e "s/VERSION =.+/VERSION = \"$version_replace\"/" setup.py
+		# sed -E --in-place='' -e "s/__version__ =.+/__version__ = \"$version_replace\"/" python2/httplib2/__init__.py python3/httplib2/__init__.py
+	echo "Updating file '$path'" >&2
+	if ! sed -E --in-place='' -e "$sed_expr" "$path" ; then
+		echo "sed error $?" >&2
+		exit 1
+	fi
+	assert_modified "$path"
+	echo "" >&2
+}
+
+edit_news() {
+	echo "Changes since last release:" >&2
+	git log --format='%h   %an   %s' "v$version"^.. -- || exit 1
+	echo "" >&2
+
+	patch -p1 <<EOT
+diff a/CHANGELOG b/CHANGELOG
+--- a/CHANGELOG
++++ b/CHANGELOG
+@@ -0,0 +1,4 @@
++$version_next
++
++  EDIT HERE. Describe important changes and link to more information.
++
+EOT
+
+	local editor=$(which edit 2>/dev/null)
+	[[ -z "$editor" ]] && editor="$EDITOR"
+	if [[ -n "$editor" ]] ; then
+		if confirm "Open default editor for CHANGELOG? [Yn] " ; then
+			$editor CHANGELOG
+		else
+			confirm "Edit CHANGELOG manually and press any key"
+		fi
+	else
+		echo "Unable to determine default text editor." >&2
+		confirm "Edit CHANGELOG manually and press any key"
+	fi
+	echo "" >&2
+
+	assert_modified CHANGELOG
+
+	echo "" >&2
+	confirm "Confirm changes? [yN] " || exit 1
+}
+
+assert_modified() {
+	local path="$1"
+	if git diff --exit-code "$path" ; then
+		echo "File '$path' is not modified" >&2
+		exit 1
+	fi
+}
+
+assert_tree_clean() {
+	if [[ -n "$(git status --short -uall)" ]] ; then
+		echo "Tree must be clean. git status:" >&2
+		echo "" >&2
+		git status --short -uall
+		echo "" >&2
+		exit 1
+	fi
+}
+
+version_check() {
+	local need=$1
+	local version_setup=$(python setup.py --version)
+	local version_py2=$(python -Es -c "$(egrep '^__version__' python2/httplib2/__init__.py);print(__version__)")
+	local version_py3=$(python -Es -c "$(egrep '^__version__' python3/httplib2/__init__.py);print(__version__)")
+	if [[ "$version_setup" != "$need" ]] ; then
+		echo "error: setup.py version=$version_setup expected=$need" >&1
+		exit 1
+	fi
+	if [[ "$version_py2" != "$need" ]] ; then
+		echo "error: python2/httplib2/__init__.py:__version__=$version_py2 expected=$need" >&1
+		exit 1
+	fi
+	if [[ "$version_py3" != "$need" ]] ; then
+		echo "error: python3/httplib2/__init__.py:__version__=$version_py3 expected=$need" >&1
+		exit 1
+	fi
+}
+
+confirm() {
+	local reply
+	local prompt="$1"
+	read -n1 -p "$prompt" reply >&2
+	echo "" >&2
+	rc=0
+	local default_y=" \[Yn\] $"
+	if [[ -z "$reply" ]] && [[ "$prompt" =~ $default_y ]] ; then
+		reply="y"
+	fi
+	[[ "$reply" != "y" ]] && rc=1
+	return $rc
+}
+
+main "$@"

gcloud auth application-default login/google-cloud-sdk/platform/gsutil/third_party/httplib2/script/test

@@ -0,0 +1,69 @@
+#!/bin/bash
+set -eux
+# By default, run tests with pytest-forked plugin,
+# disable in terminal for debugging, you may add --forked
+flag_forked="--forked"
+if [[ -z "${CI:-}" ]] && [[ -t 1 ]] ; then
+	flag_forked=""
+fi
+test_flags=(
+	$@
+	$flag_forked
+	tests/
+)
+
+main() {
+	cd "$( dirname "${BASH_SOURCE[0]}" )/.."
+	if [[ -n "${CI:-}" ]] ; then
+		case "${test_group-}" in
+		package)
+			# TODO: sdist bdist_wheel
+			# but wheels don't roll well with our 2/3 split code base
+			python setup.py sdist
+			install_check_version "pip"
+			;;
+		*)
+			pip install -e .
+			httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_303 $@ tests/ || true
+			httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_head_301 $@ tests/ || true
+			pytest --fulltrace ${test_flags[@]}
+			;;
+		esac
+	else
+		if [[ ! -d ./venv-27 ]] ; then
+			virtualenv --python=python2.7 ./venv-27
+		fi
+		if [[ ! -d ./venv-36 ]] ; then
+			virtualenv --python=python3.6 ./venv-36
+		fi
+
+		./venv-27/bin/pip install -e . -r requirements-test.txt
+		./venv-27/bin/pytest ${test_flags[@]}
+		./venv-36/bin/pip install -e . -r requirements-test.txt
+		./venv-36/bin/pytest ${test_flags[@]}
+
+		# FIXME: too many errors
+		# ./venv-27/bin/flake8 python2/
+		# ./venv-36/bin/flake8 python3/ tests/
+
+		# TODO: sdist bdist_wheel
+		# but wheels don't roll well with our 2/3 split code base
+		./venv-36/bin/python setup.py sdist
+		install_check_version "./venv-27/bin/pip"
+		install_check_version "./venv-36/bin/pip"
+	fi
+	rm -rf ./_httplib2_test_cache
+}
+
+install_check_version() {
+	local pip="$1"
+	$pip install dist/httplib2*
+	version_source=$(python setup.py --version)
+	version_installed=$($pip show httplib2 |fgrep Version: |cut -d' ' -f2)
+	if [[ "$version_source" != "$version_installed" ]] ; then
+		echo "error: installed package version=$version_installed does not match source=$version_source" >&2
+		exit 1
+	fi
+}
+
+main "$@"