if ! ps -p $pid1 1>&2 >/dev/null; then
		echo "server1 log:"
		cat "${WORK_DIR}/dist-minio-server1.log"
		echo "FAILED"
		purge "$WORK_DIR"
		exit 1
	fi

	if ! ps -p $pid2 1>&2 >/dev/null; then
		echo "server2 log:"
		cat "${WORK_DIR}/dist-minio-server2.log"
		echo "FAILED"
		purge "$WORK_DIR"
		exit 1
	fi

	if ! ps -p $pid3 1>&2 >/dev/null; then
		echo "server3 log:"

OLD_BINARY_LINK=https://dl.min.io/server/minio/release/linux-amd64/archive/minio.${OLD_VERSION}

__init__() {
	if which curl &>/dev/null; then
		echo "curl is already installed"
	else
		echo "Installing curl:"
		sudo apt install curl -y
	fi

	export GOPATH=/tmp/gopath
	export PATH="${PATH}":"${GOPATH}"/bin

	if which mc &>/dev/null; then
		echo "mc is already installed"
	else
		echo "Installing mc:"
		go install github.com/minio/mc@latest

		purge "${MC_BUILD_DIR}"
	fi

	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" >"${WORK_DIR}/server1.log" 2>&1 &
	pid=$!
	disown $pid
	sleep 5

	if ! ps -p ${pid} 1>&2 >/dev/null; then
		echo "server1 log:"
		cat "${WORK_DIR}/server1.log"
		echo "FAILED"
		purge "$WORK_DIR"
		exit 1
	fi

	"${PWD}/mc" mb --with-versioning minio/bucket

	for i in $(seq 1 4); do

		-f "buildscripts/upgrade-tests/compose.yml" \
		rm -s -f
	docker volume prune -f
}

verify_checksum_after_heal() {
	local sum1
	sum1=$(curl -s "$2" | sha256sum)
	mc admin heal --json -r "$1" >/dev/null # test after healing
	local sum1_heal
	sum1_heal=$(curl -s "$2" | sha256sum)

	if [ "${sum1_heal}" != "${sum1}" ]; then
		echo "mismatch expected ${sum1_heal}, got ${sum1}"
		exit 1
	fi
}

		exit 1
	fi

	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")

	# remove mc source.
	purge "${MC_BUILD_DIR}"

	shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
	shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null

	## version is purposefully set to '3' for minio to migrate configuration file

	sleep "$1"

	if ! ps -p $pid1 1>&2 >/dev/null; then
		echo "server1 log:"
		cat "${WORK_DIR}/dist-minio-server1.log"
		echo "FAILED"
		purge "$WORK_DIR"
		exit 1
	fi

	if ! ps -p $pid2 1>&2 >/dev/null; then
		echo "server2 log:"
		cat "${WORK_DIR}/dist-minio-server2.log"
		echo "FAILED"
		purge "$WORK_DIR"
		exit 1
	fi

	if ! ps -p $pid3 1>&2 >/dev/null; then
		echo "server3 log:"

	# remove mc source.
	purge "${MC_BUILD_DIR}"

	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
	pid=$!
	disown $pid
	sleep 10

	if ! ps -p ${pid} 1>&2 >/dev/null; then
		echo "server1 log:"
		cat "${WORK_DIR}/server1.log"
		echo "FAILED"
		purge "$WORK_DIR"
		exit 1
	fi

	"${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
	"${WORK_DIR}/mc" cp \

./mc admin policy create minio1 rw ./docs/site-replication/rw.json

sleep 5
./mc admin policy info minio2 rw >/dev/null 2>&1

./mc admin replicate status minio1

## Add a new empty site
./mc admin replicate add minio1 minio2 minio3

sleep 10

./mc admin policy info minio3 rw >/dev/null 2>&1

./mc admin policy remove minio3 rw

./mc admin replicate status minio3

sleep 10

./mc admin user info minio3 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
./mc admin policy create minio1 rw ./docs/site-replication/rw.json

sleep 5
./mc admin policy info minio2 rw >/dev/null 2>&1
./mc admin policy info minio3 rw >/dev/null 2>&1

./mc admin policy remove minio3 rw

sleep 10
./mc admin policy info minio1 rw
if [ $? -eq 0 ]; then
	echo "expecting the command to fail, exiting.."
	exit_1
fi

	for i in $(seq 1 4); do
		"${MINIO[@]}" --address ":$((start_port + i))" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
	done

	# Wait until all nodes return 403
	for i in $(seq 1 4); do
		while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$((start_port + i)))" -ne "403" ]; do
			echo -n "."
			sleep 1
		done
	done

}

# Prepare fake disks with losetup
function prepare_block_devices() {
	set -e