// The first selection prioritizes the non-failed routes.
    val selection2 = routeSelector.next()
    assertThat(selection2.next()).isEqualTo(regularRoutes[1])
    assertThat(selection2.hasNext()).isFalse()

    // The second selection will contain all failed routes.
    val selection3 = routeSelector.next()
    assertThat(selection3.next()).isEqualTo(regularRoutes[0])

   */
  open fun dispatcherQueueEnd(
    call: Call,
    dispatcher: Dispatcher,
  ) {
  }

  /**
   * Invoked prior to a proxy selection.
   *
   * This will be invoked for route selection regardless of whether the client
   * is configured with a single proxy, a proxy selector, or neither.
   *
   * @param url a URL with only the scheme, hostname, and port specified.
   */

   */
  operator fun hasNext(): Boolean = hasNextProxy() || postponedRoutes.isNotEmpty()

  @Throws(IOException::class)
  operator fun next(): Selection {
    if (!hasNext()) throw NoSuchElementException()

    // Compute the next set of routes to attempt.
    val routes = mutableListOf<Route>()
    while (hasNextProxy()) {

```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM
**UrlConvertHelper**: URL normalization

---

## Development Workflow

### Build Commands

```bash
mvn clean install              # Build all

| `LanguageHelper` | `org.codelibs.fess.helper` | Language detection and validation |
| `SystemHelper` | `org.codelibs.fess.helper` | UI language dropdown builder |
| `FessUserLocaleProcessProvider` | `org.codelibs.fess.mylasta.direction.sponsor` | User locale detection |

## Language Detection and Selection

### User Interface Language

Fess determines the UI language in this order:

 * This class implements AutoCloseable to properly handle resource cleanup.
 *
 * <p>The factory maintains a map of regular expression patterns to crawler clients,
 * allowing for URL-based client selection. Clients can be added with specific patterns
 * and optionally at specific positions in the processing order.</p>
 *
 * <p>This factory is typically initialized through dependency injection and can be

  private val routeDatabase: RouteDatabase,
  private val call: RealCall,
  request: Request,
) : RoutePlanner {
  private val doExtensiveHealthChecks = request.method != "GET"
  private var routeSelection: RouteSelector.Selection? = null
  private var routeSelector: RouteSelector? = null
  private var nextRouteToTry: Route? = null

  override val deferredPlans = ArrayDeque<Plan>()

  override fun isCanceled(): Boolean = call.isCanceled()

  fun recordedConnectionEventTypes(): List<String> = connectionListener.recordedEventTypes()

  companion object {
    /**
     * A network that resolves only one IP address per host. Use this when testing route selection
     * fallbacks to prevent the host machine's various IP addresses from interfering.
     */
    private val SINGLE_INET_ADDRESS_DNS =
      Dns { hostname ->
        val addresses = Dns.SYSTEM.lookup(hostname)

 * <li>Document expiration handling</li>
 * <li>Child URL extraction and queueing</li>
 * <li>Integration with Fess configuration and permission systems</li>
 * <li>Client selection based on URL patterns</li>
 * </ul>
 *
 * @see CrawlerThread
 * @see org.codelibs.fess.crawler.client.CrawlerClient
 */
public class FessCrawlerThread extends CrawlerThread {

    /**

     *
     * This method prioritizes explicit language parameters over browser locale settings
     * and handles special cases like "all languages" selection.
     *
     * @param request The HTTP servlet request
     * @param params The search request parameters
     * @return Array of normalized language codes
     */