MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
GOPATH=/tmp/gopath

function start_minio_3_node() {
	export MINIO_ROOT_USER=minio
	export MINIO_ROOT_PASSWORD=minio123
	export MINIO_ERASURE_SET_DRIVE_COUNT=6
	export MINIO_CI_CD=1

	first_time=$(find ${WORK_DIR}/ | grep format.json | wc -l)

	start_port=$2
	args=""
	for d in $(seq 1 3 5); do

# Prepare fake disks with losetup
function prepare_block_devices() {
	set -e
	mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
	sudo modprobe loop
	for i in 1 2 3 4; do
		dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
		device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
		sudo mkfs.ext4 -F ${device}
		mkdir -p ${WORK_DIR}/mnt/disk${i}/
		sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/

MINIO_CONFIG_DIR="$WORK_DIR/.minio"
MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)

function start_minio_3_node() {
	export MINIO_ROOT_USER=minio
	export MINIO_ROOT_PASSWORD=minio123
	export MINIO_ERASURE_SET_DRIVE_COUNT=6
	export MINIO_CI_CD=1

	start_port=$2
	args=""
	for i in $(seq 1 3); do

	echo "BUG: Expected ILM expiry replication not set for 'siteb'"
	exit 1
fi

## Check if ILM expiry rules replicated
sleep 30s

./mc ilm rule list siteb/bucket
count=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules | length')
if [ $count -ne 1 ]; then
	echo "BUG: ILM expiry rules not replicated to 'siteb'"
	exit 1
fi

## Check replication of rules content

pid_2=$!

sleep 30

expanded_user_count=$(./mc admin user list myminio/ | wc -l)
expanded_policy_count=$(./mc admin policy list myminio/ | wc -l)

if [ $user_count -ne $expanded_user_count ]; then
	echo "BUG: original user count differs from expanded setup"
	exit 1
fi

if [ $policy_count -ne $expanded_policy_count ]; then
	echo "BUG: original policy count  differs from expanded setup"
	exit 1
fi

sitea_out=$(cat /tmp/sitea_dirs.txt)
siteb_out=$(cat /tmp/siteb_dirs.txt)

if [ $sitea_count -ne 0 ]; then
	echo "BUG: expected no 'directory objects' left after deletion: ${sitea_out}"
	exit 1
fi

if [ $siteb_count -ne 0 ]; then
	echo "BUG: expected no 'directory objects' left after deletion: ${siteb_out}"
	exit 1
fi

## upgrading to master, healing and being able to recover
## the last version.
if [ $failed_count_site1 -ne 0 ]; then
	echo "failed with multipart on site1 uploads ${failed_count_site1}"
fi

if [ $failed_count_site2 -ne 0 ]; then
	echo "failed with multipart on site2 uploads ${failed_count_site2}"
fi

export RELEASE=${1}

docker-compose -f docker-compose-site1.yaml up -d

sleep 1

./mc ls -r --versions myminio1/testbucket/dir/ >/tmp/dir_1.txt
./mc ls -r --versions myminio2/testbucket/dir/ >/tmp/dir_2.txt

out=$(diff -qpruN /tmp/dir_1.txt /tmp/dir_2.txt)
ret=$?
if [ $ret -ne 0 ]; then
	echo "BUG: expected no 'diff' after replication: $out"
	exit 1

upload_id=$(multipart-debug --endpoint 127.0.0.1:9001 --accesskey minio --secretkey minio123 multipart new --bucket bucket --object new-test-encrypted-object --encrypt)

dd if=/dev/urandom bs=1 count=7048531 of=/tmp/7048531.txt
dd if=/dev/urandom bs=1 count=2847391 of=/tmp/2847391.txt

sudo apt install jq -y

sleep 30

./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt
./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt

out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt)
ret=$?
if [ $ret -ne 0 ]; then
	echo "BUG: expected no missing entries after replication resync: $out"
	exit 1