private final BlockingQueue<HttpUrl> queue;
  private final ConcurrentHashMap<String, AtomicInteger> hostnames = new ConcurrentHashMap<>();
  private final int hostLimit;

  public Crawler(OkHttpClient client, int queueLimit, int hostLimit) {
    this.client = client;
    this.queue = new LinkedBlockingQueue<>(queueLimit);
    this.hostLimit = hostLimit;
  }

  @CanIgnoreReturnValue
  public static Set<Feature<?>> addImpliedFeatures(Set<Feature<?>> features) {
    Queue<Feature<?>> queue = new ArrayDeque<>(features);
    while (!queue.isEmpty()) {
      Feature<?> feature = queue.remove();
      for (Feature<?> implied : feature.getImpliedFeatures()) {
        if (features.add(implied)) {
          queue.add(implied);
        }
      }
    }
    return features;
  }

  /**

  @CanIgnoreReturnValue
  public static Set<Feature<?>> addImpliedFeatures(Set<Feature<?>> features) {
    Queue<Feature<?>> queue = new ArrayDeque<>(features);
    while (!queue.isEmpty()) {
      Feature<?> feature = queue.remove();
      for (Feature<?> implied : feature.getImpliedFeatures()) {
        if (features.add(implied)) {
          queue.add(implied);
        }
      }
    }
    return features;
  }

  /**

      implements PeekingIterator<T> {
    private final Queue<T> queue;

    BreadthFirstIterator(T root) {
      this.queue = new ArrayDeque<>();
      queue.add(root);
    }

    @Override
    public boolean hasNext() {
      return !queue.isEmpty();
    }

    @Override
    public T peek() {
      return queue.element();
    }

    @Override
    public T next() {

import com.google.common.annotations.GwtCompatible;
import com.google.common.collect.testing.SampleElements.Strings;
import java.util.List;
import java.util.Queue;
import org.jspecify.annotations.NullMarked;

/**
 * Create queue of strings for tests.
 *
 * @author Jared Levy
 */
@GwtCompatible
@NullMarked
public abstract class TestStringQueueGenerator implements TestQueueGenerator<String> {
  @Override

        }
        log(logHelper, LogType.FINISHED_THREAD, crawlerContext);
    }

    /**
     * Adds sitemaps from robots.txt to the crawling queue.
     * @param urlQueue The URL queue to add sitemaps to.
     */
    protected void addSitemapsFromRobotsTxt(final UrlQueue<?> urlQueue) {
        final String[] sitemaps = crawlerContext.removeSitemaps();
        if (sitemaps != null) {

    private static final int MAX_BUFFER_SIZE = 0x100000; // 1MB maximum

    // Use concurrent queue for lock-free operations
    private static final java.util.concurrent.ConcurrentLinkedQueue<byte[]> bufferQueue =
            new java.util.concurrent.ConcurrentLinkedQueue<>();

    // Track queue size with atomic counter for efficiency

* Add validation for CustomResources via JSON Schema. ([#47263](https://github.com/kubernetes/kubernetes/pull/47263), [@nikhita](https://github.com/nikhita))
* enqueue a sync task to wake up jobcontroller to check job ActiveDeadlineSeconds in time ([#48454](https://github.com/kubernetes/kubernetes/pull/48454), [@weiwei04](https://github.com/weiwei04))

 *   <li>Before processing a URL ({@link #delayBeforeProcessing()})</li>
 *   <li>After processing a URL ({@link #delayAfterProcessing()})</li>
 *   <li>When there are no URLs in the queue ({@link #delayAtNoUrlInQueue()})</li>
 *   <li>While waiting for new URLs to be added to the queue ({@link #delayForWaitingNewUrl()})</li>
 * </ul>
 *
 * <p>
 * Subclasses are responsible for implementing the abstract methods to define the actual delay

 * and when waiting for new URLs.
 * The delays are configurable via constructor parameters.
 *
 */
public class DefaultIntervalController extends AbstractIntervalController {

    /** Delay in milliseconds after processing a URL */
    protected long delayMillisAfterProcessing = 0L;

    /** Delay in milliseconds when no URL is in the queue */