}

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */

        t.numThreads = 5;
        t.numIter = 10;
        t.delay = 250L;
        t.url = "smb://custom/path/file.txt";

        // Verify values are set
        assertEquals(5, t.numThreads, "numThreads should be configurable");
        assertEquals(10, t.numIter, "numIter should be configurable");
        assertEquals(250L, t.delay, "delay should be configurable");

 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.
 * If the time since the last request to the host is less than the configured delay,
 * the thread waits until the delay has elapsed.
 * This class is thread-safe.
 */

        this.delay = delay;
        this.unit = checkNotNull(unit);
      }

      /**
       * @param delay the time from now to delay execution
       * @since 31.1 (but only since 33.4.0 in the Android flavor)
       */
      public Schedule(Duration delay) {
        this(toNanosSaturated(delay), NANOSECONDS);
      }
    }

    /**

    long lastDelay;
    TimeUnit lastUnit;

    void assertLastMethodCalled(String method, long delay, TimeUnit unit) {
      assertEquals(method, lastMethodCalled);
      assertEquals(delay, lastDelay);
      assertEquals(unit, lastUnit);
    }

    void assertLastMethodCalled(String method, long initialDelay, long delay, TimeUnit unit) {
      assertEquals(method, lastMethodCalled);

User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

	lastObject := ri.Object

	retryAttempts := job.KeyRotate.Flags.Retry.Attempts
	if retryAttempts <= 0 {
		retryAttempts = batchKeyRotateJobDefaultRetries
	}
	delay := job.KeyRotate.Flags.Retry.Delay
	if delay <= 0 {
		delay = batchKeyRotateJobDefaultRetryDelay
	}

	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))

	selectObj := func(info FileInfo) (ok bool) {

  }

  @Override
  public final ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
    return delegate.schedule(wrapTask(command), delay, unit);
  }

  @Override
  public final <V extends @Nullable Object> ScheduledFuture<V> schedule(
      Callable<V> task, long delay, TimeUnit unit) {
    return delegate.schedule(wrapTask(task), delay, unit);
  }

  @Override
  public final ScheduledFuture<?> scheduleAtFixedRate(

                    // Check for potential overflow before multiplication
                    long delay = (retryDelayMs > Long.MAX_VALUE / multiplier) ? MAX_RETRY_DELAY
                            : Math.min(MAX_RETRY_DELAY, retryDelayMs * multiplier);
                    Thread.sleep(delay);
                }

                // Attempt to reset the connection
                connection.reset();

enabled by default. The following configuration settings allow for more staggered delay in terms of usage calculation. The scanner adapts to the system speed and completely pauses when the system is under load. It is possible to adjust the speed of the scanner and thereby the latency of updates being reflected. The delays between each operation of the scanner can be adjusted by the `mc admin config set alias/ delay=15.0`. By default the value is `10.0`. This means the scanner will sleep *10x* the...