--remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \
	--replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
sleep 1

echo "adding replication rule for b -> a : ${remote_arn}"
./mc replicate add siteb/bucket/ \
	--remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \
	--replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
sleep 1

echo "adding replication rule for a -> c : ${remote_arn}"

out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt)
ret=$?
if [ $ret -ne 0 ]; then
	echo "BUG: expected no missing entries after replication: $out"
	exit 1
fi

./mc cp /tmp/data/file_1.txt sitea/bucket/marker_new
./mc rm sitea/bucket/marker_new

sleep 12s ## sleep for 12s idea is that we give 100ms per object.

./mc ls -r --versions sitea/bucket >/tmp/sitea.txt

# create a replication policy for repluser
# Remove "s3:GetBucketObjectLockConfiguration" if object locking is not enabled, i.e. bucket was not created with `mc mb --with-lock` option
# Remove "s3:ReplicateDelete" if delete marker replication is not required
cat >replpolicy.json <<EOF
{
 "Version": "2012-10-17",
 "Statement": [
  {
   "Effect": "Allow",
   "Action": [
    "s3:GetReplicationConfiguration",
    "s3:ListBucket",

failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)

## we do not need to fail here, since we are going to test
## upgrading to master, healing and being able to recover
## the last version.
if [ $failed_count_site1 -ne 0 ]; then
	echo "failed with multipart on site1 uploads ${failed_count_site1}"
fi

if [ $failed_count_site2 -ne 0 ]; then