CrawlingParameterUtil.setUrlQueue(q);

        // First access - should not delay
        final long start1 = System.nanoTime();
        controller.delayBeforeProcessing();
        final long elapsed1 = (System.nanoTime() - start1) / 1000000;
        assertTrue("First access should not delay", elapsed1 < 50);

        // Second access to same host - should delay
        q.setUrl("http://example.com/page2");

   * Creates an {@code MonitorBasedArrayBlockingQueue} with the given (fixed) capacity and the
   * specified access policy.
   *
   * @param capacity the capacity of this queue
   * @param fair if {@code true} then queue accesses for threads blocked on insertion or removal,
   *     are processed in FIFO order; if {@code false} the access order is unspecified.
   * @throws IllegalArgumentException if {@code capacity} is less than 1
   */

        if (logHelper != null) {
            logHelper.log(key, objs);
        }
    }

    /**
     * Runs the crawling process in a separate thread.
     * This method fetches URLs from the queue, accesses content, processes responses,
     * and extracts child URLs until the crawling process is done or no more URLs are available.
     */
    @Override
    public void run() {

import okhttp3.internal.tls.OkHostnameVerifier
import okio.Buffer

/**
 * A connection to a remote web server capable of carrying 1 or more concurrent streams.
 *
 * Connections are shared in a connection pool. Accesses to the connection's state must be guarded
 * by holding a lock on the connection.
 */
class RealConnection internal constructor(
  val taskRunner: TaskRunner,
  val connectionPool: RealConnectionPool,

 * @author Jon Noack
 */
@GwtIncompatible // not worth using in GWT for now
class CompactHashSet<E extends @Nullable Object> extends AbstractSet<E> implements Serializable {
  // TODO(user): cache all field accesses in local vars

  /** Creates an empty {@code CompactHashSet} instance. */
  public static <E extends @Nullable Object> CompactHashSet<E> create() {
    return new CompactHashSet<>();
  }

  /**

   *
   *   o CLEAN lines track a cache entry that has been successfully published and may be read. A
   *     publish line is followed by the lengths of each of its values.
   *
   *   o READ lines track accesses for LRU.
   *
   *   o REMOVE lines track entries that have been deleted.
   *
   * The journal file is appended to as cache operations occur. The journal may occasionally be

            }

            try {
                final Builder builder = MinioClient.builder().endpoint(endpoint);
                if (StringUtil.isNotBlank(accessKey) && StringUtil.isNotBlank(secretKey)) {
                    builder.credentials(accessKey, secretKey);
                }
                if (StringUtil.isNotBlank(region)) {
                    builder.region(region);
                }

        }
        final String accessKey = getInitParameter("accessKey", null, String.class);
        if (StringUtil.isBlank(accessKey)) {
            throw new CrawlingAccessException(
                    "S3 access key is blank. Please set the S3_ACCESS_KEY environment variable or accessKey parameter.");
        }

        }
        builder.endpoint(endpoint);
        final String accessKey = getInitParameter("accessKey", null, String.class);
        if (StringUtil.isBlank(accessKey)) {
            throw new CrawlingAccessException(
                    "Storage access key is blank. Please set the STORAGE_ACCESS_KEY environment variable or accessKey parameter.");
        }

     *
     * @param endpoint the S3 endpoint URL (null for AWS default)
     * @param accessKey the AWS access key
     * @param secretKey the AWS secret key
     * @param bucket the bucket name
     * @param region the AWS region
     */
    public S3StorageClient(final String endpoint, final String accessKey, final String secretKey, final String bucket,
            final String region) {
        this.bucket = bucket;