failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)

## we do not need to fail here, since we are going to test
## upgrading to master, healing and being able to recover
## the last version.
if [ $failed_count_site1 -ne 0 ]; then
	echo "failed with multipart on site1 uploads ${failed_count_site1}"

ret=$?
if [ $ret -ne 0 ]; then
	echo "BUG: expected no 'diff' after replication: $out"
	exit 1
fi

sitea_count=$(cat /tmp/sitea_dirs.txt | wc -l) # need to do it this way to avoid filename in the output
siteb_count=$(cat /tmp/siteb_dirs.txt | wc -l) # need to do it this way to avoid filename in the output
sitea_out=$(cat /tmp/sitea_dirs.txt)
siteb_out=$(cat /tmp/siteb_dirs.txt)

if [ $sitea_count -ne 0 ]; then