User-agent: FessCrawler
Disallow:           # allows all 

User-agent: BruteBot
Disallow: /
Allow: /foo/bar/
Crawl-delay: 1314000

# welcome!
User-agent: Googlebot
Crawl-delay: 1

User-agent: *
Disallow: /private/
Disallow: /help        # disallows /help.html, /help/index.html, etc.
Allow: /help/faq.html
Crawl-delay: 3

User-agent: Crawler
Disallow: /aaa

User-agent: Crawler/1.0
Disallow: /bbb

        t.numThreads = 5;
        t.numIter = 10;
        t.delay = 250L;
        t.url = "smb://custom/path/file.txt";

        // Verify values are set
        assertEquals(5, t.numThreads, "numThreads should be configurable");
        assertEquals(10, t.numIter, "numIter should be configurable");
        assertEquals(250L, t.delay, "delay should be configurable");

 *
 * <p>
 * Subclasses are responsible for implementing the abstract methods to define the actual delay
 * mechanism for each of these stages.
 * </p>
 *
 * <p>
 * The class also provides a mechanism to ignore exceptions that may occur during the delay process.
 * If {@link #ignoreException} is set to true, any exceptions thrown during the delay will be caught
 * and ignored. Otherwise, they will be re-thrown as {@link CrawlerSystemException}.

    long lastDelay;
    TimeUnit lastUnit;

    void assertLastMethodCalled(String method, long delay, TimeUnit unit) {
      assertEquals(method, lastMethodCalled);
      assertEquals(delay, lastDelay);
      assertEquals(unit, lastUnit);
    }

    void assertLastMethodCalled(String method, long initialDelay, long delay, TimeUnit unit) {
      assertEquals(method, lastMethodCalled);

        this.delay = delay;
        this.unit = checkNotNull(unit);
      }

      /**
       * @param delay the time from now to delay execution
       * @since 31.1 (but only since 33.4.0 in the Android flavor)
       */
      public Schedule(Duration delay) {
        this(toNanosSaturated(delay), NANOSECONDS);
      }
    }

    /**

  }

  @Override
  public final ScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
    return delegate.schedule(wrapTask(command), delay, unit);
  }

  @Override
  public final <V extends @Nullable Object> ScheduledFuture<V> schedule(
      Callable<V> task, long delay, TimeUnit unit) {
    return delegate.schedule(wrapTask(task), delay, unit);
  }

  @Override
  public final ScheduledFuture<?> scheduleAtFixedRate(

 * The delayBeforeProcessing method is overridden to introduce a delay before processing a URL,
 * ensuring that requests to the same host are not made too frequently.
 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.
 * If the time since the last request to the host is less than the configured delay,
 * the thread waits until the delay has elapsed.
 * This class is thread-safe.

      delay(SMALL_DELAY_MS);
      return Boolean.TRUE;
    }
  }

  public class MediumRunnable extends CheckedRunnable {
    @Override
    protected void realRun() throws Throwable {
      delay(MEDIUM_DELAY_MS);
    }
  }

  public class MediumInterruptedRunnable extends CheckedInterruptedRunnable {
    @Override
    protected void realRun() throws InterruptedException {
      delay(MEDIUM_DELAY_MS);

Isso significa que no lugar do processo normal de seus usuários enviarem requisições para a sua API, é a **sua API** (ou sua aplicação) que poderia **enviar requisições para o sistema deles** (para a API deles, a aplicação deles).

Isso normalmente é chamado de **webhook**.

                    // Check for potential overflow before multiplication
                    long delay = (retryDelayMs > Long.MAX_VALUE / multiplier) ? MAX_RETRY_DELAY
                            : Math.min(MAX_RETRY_DELAY, retryDelayMs * multiplier);
                    Thread.sleep(delay);
                }

                // Attempt to reset the connection
                connection.reset();