Force      bool             // Force deletion
	SRDeleteOp SRBucketDeleteOp // only when site replication is enabled
}

// BucketOptions provides options for ListBuckets and GetBucketInfo call.
type BucketOptions struct {
	Deleted    bool // true only when site replication is enabled

		return
	}

	ot, err := objAPI.GetObjectTags(ctx, bucket, object, opts)
	if err != nil {
		// if object/version is not found locally, but exists on peer site - proxy
		// the tagging request to peer site. The response to client will
		// return tags from peer site.
		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
			proxytgts := getProxyTargets(ctx, bucket, object, opts)
			if !proxytgts.Empty() {

        // Simulate crawling progress
        for (int i = 0; i < 100; i++) {
            crawlerContext.incrementAndGetAccessCount();
            crawlerContext.getRobotsTxtUrlSet().add("http://site" + i + ".com/robots.txt");
        }

        // Add sitemaps
        crawlerContext.addSitemaps(new String[] { "http://example.com/sitemap.xml" });

        // Complete crawling