```

### CrawlerThread (`CrawlerThread.java`)

Worker thread for crawling.

**Flow**: Poll URL → Validate → Get client → Delay → Check last-modified → Execute → Process → Extract children → Queue children → Delay

### CrawlerClientFactory

Pattern-based client selection using `LinkedHashMap<Pattern, CrawlerClient>`.

**Standard Patterns**:
```java

                        crawlerContext.intervalController.delay(IntervalController.NO_URL_IN_QUEUE);
                    }

                    threadCheckCount++;
                }

                // interval
                if (crawlerContext.intervalController != null) {
                    crawlerContext.intervalController.delay(IntervalController.WAIT_NEW_URL);
                }
            }

	  token: "Bearer xxxxx" # optional authentication token for the notification endpoint

	retry:
	  attempts: 10 # number of retries for the job before giving up
	  delay: "500ms" # least amount of delay between each retry
```

You can create and run multiple 'replication' jobs at a time there are no predefined limits set.

## Batch Jobs Terminology

### Job

	}
	globalBatchJobsMetrics.save(job.ID, ri)

	retryAttempts := job.Replicate.Flags.Retry.Attempts
	if retryAttempts <= 0 {
		retryAttempts = batchReplJobDefaultRetries
	}
	delay := job.Replicate.Flags.Retry.Delay
	if delay <= 0 {
		delay = batchReplJobDefaultRetryDelay
	}
	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))

	hasTags := len(r.Flags.Filter.Tags) != 0
	isMetadata := len(r.Flags.Filter.Metadata) != 0

      "FINE: Q10000 run again after 150 µs: task",
      "FINE: Q10000 starting              : task",
      "FINE: Q10000 finished run in   0 µs: task",
    )
  }

  /** Repeat with a delay of 200 but schedule with a delay of 50. The schedule wins. */
  @Test fun executeScheduledEarlierReplacesRepeatedLater() {
    val task =
      object : Task("task") {
        val schedules = mutableListOf(50.µs)

    }

    @Override
    public ListenableScheduledFuture<?> schedule(Runnable command, long delay, TimeUnit unit) {
      TrustedListenableFutureTask<@Nullable Void> task =
          TrustedListenableFutureTask.create(command, null);
      ScheduledFuture<?> scheduled = delegate.schedule(task, delay, unit);
      return new ListenableScheduledTask<@Nullable Void>(task, scheduled);
    }

    @Override

		},
		{
			val:   0.79,
			label: labels[2],
		},
	}
	ticker := time.NewTicker(1 * time.Millisecond)
	defer ticker.Stop()
	for _, obs := range observations {
		// Send observations once every 1ms, to simulate delay between
		// observations. This is to test the channel based
		// synchronization used internally.
		<-ticker.C
		ttfbHist.With(prometheus.Labels{"api": obs.label}).Observe(obs.val)
	}

Pt.indexOf(t)&&delete e[t]})),"number"==typeof(t=r({},this.constructor.Default,e,"object"==typeof t&&t?t:{})).delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),d.typeCheckConfig(It,t,this.constructor.DefaultType),t.sanitize&&(t.template=At(t.template,t.whiteList,t.sanitizeFn)),t},e._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]...

        // Minimise test flakiness due to possible race conditions with connections closing.
        // Some number of tests will report here, but not fail due to this delay.
        println("Delaying to avoid flakes")
        Thread.sleep(500L)
        println("After delay: " + connectionPool.connectionCount())
      }

      connectionPool.evictAll()
      assertEquals(0, connectionPool.connectionCount()) {

    }

    @Test
    public void testRdmaErrorHandlerBehavior() {
        RdmaStatistics stats = new RdmaStatistics();
        RdmaErrorHandler errorHandler = new RdmaErrorHandler(stats, 3, 50); // 3 retries, 50ms delay

        assertNotNull(errorHandler, "Error handler should not be null");

        // Test error classification for different exception types