MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
GOPATH=/tmp/gopath

function start_minio_3_node() {
	export MINIO_ROOT_USER=minio
	export MINIO_ROOT_PASSWORD=minio123
	export MINIO_ERASURE_SET_DRIVE_COUNT=6
	export MINIO_CI_CD=1

	first_time=$(find ${WORK_DIR}/ | grep format.json | wc -l)

	start_port=$2
	args=""
	for d in $(seq 1 3 5); do

./mc ls rootsvc
if [ $? -ne 0 ]; then
	echo "root service account not inherited root permissions, exiting.."
	exit_1
fi

./mc admin user svcacct info minio1 testsvc
if [ $? -ne 0 ]; then
	echo "svc account not mirrored, exiting.."
	exit_1
fi

./mc admin user svcacct info minio2 testsvc
if [ $? -ne 0 ]; then
	echo "svc account not mirrored, exiting.."
	exit_1
fi

./mc admin user svcacct info minio1 testsvc
if [ $? -ne 0 ]; then
	echo "svc account not mirrored, exiting.."
	exit_1
fi

./mc admin user svcacct info minio2 testsvc
if [ $? -ne 0 ]; then
	echo "svc account not mirrored, exiting.."
	exit_1
fi

./mc admin user svcacct info minio3 testsvc
if [ $? -ne 0 ]; then
	echo "svc account not mirrored, exiting.."
	exit_1
fi

# Prepare fake disks with losetup
function prepare_block_devices() {
	set -e
	mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
	sudo modprobe loop
	for i in 1 2 3 4; do
		dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
		device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
		sudo mkfs.ext4 -F ${device}
		mkdir -p ${WORK_DIR}/mnt/disk${i}/
		sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/

MINIO_CONFIG_DIR="$WORK_DIR/.minio"
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)

function start_minio_3_node() {
	export MINIO_ROOT_USER=minio
	export MINIO_ROOT_PASSWORD=minio123
	export MINIO_ERASURE_SET_DRIVE_COUNT=6
	export MINIO_CI_CD=1

	start_port=$2
	args=""
	for i in $(seq 1 3); do

	echo "BUG: Expected ILM expiry replication not set for 'siteb'"
	exit 1
fi

## Check if ILM expiry rules replicated
sleep 30s

./mc ilm rule list siteb/bucket
count=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules | length')
if [ $count -ne 1 ]; then
	echo "BUG: ILM expiry rules not replicated to 'siteb'"
	exit 1
fi

## Check replication of rules content

STS_ACCESS_KEY=$(echo ${STS_CRED} | cut -d ':' -f 1)

# Create service account for STS user
./mc admin user svcacct add minio2 $STS_ACCESS_KEY --access-key testsvc --secret-key testsvc123
if [ $? -ne 0 ]; then
	echo "adding svc account failed, exiting.."
	exit_1
fi

sleep 10

./mc admin user svcacct info minio1 testsvc
if [ $? -ne 0 ]; then
	echo "svc account not mirrored, exiting.."
	exit_1
fi

pid_2=$!

sleep 30

expanded_user_count=$(./mc admin user list myminio/ | wc -l)
expanded_policy_count=$(./mc admin policy list myminio/ | wc -l)

if [ $user_count -ne $expanded_user_count ]; then
	echo "BUG: original user count differs from expanded setup"
	exit 1
fi

if [ $policy_count -ne $expanded_policy_count ]; then
	echo "BUG: original policy count  differs from expanded setup"
	exit 1
fi

## upgrading to master, healing and being able to recover
## the last version.
if [ $failed_count_site1 -ne 0 ]; then
	echo "failed with multipart on site1 uploads ${failed_count_site1}"
fi

if [ $failed_count_site2 -ne 0 ]; then
	echo "failed with multipart on site2 uploads ${failed_count_site2}"
fi

export RELEASE=${1}

docker-compose -f docker-compose-site1.yaml up -d

	exit 1
fi

sitea_count=$(cat /tmp/sitea_dirs.txt | wc -l) # need to do it this way to avoid filename in the output
siteb_count=$(cat /tmp/siteb_dirs.txt | wc -l) # need to do it this way to avoid filename in the output
sitea_out=$(cat /tmp/sitea_dirs.txt)
siteb_out=$(cat /tmp/siteb_dirs.txt)

if [ $sitea_count -ne 0 ]; then