}

    /**
     * Returns the set of robots.txt URLs.
     * @return The set of robots.txt URLs.
     */
    public Set<String> getRobotsTxtUrlSet() {
        return robotsTxtUrlSet;
    }

    /**
     * Sets the set of robots.txt URLs.
     * @param robotsTxtUrlSet The set of robots.txt URLs.
     */
    public void setRobotsTxtUrlSet(final Set<String> robotsTxtUrlSet) {

/**
 * The search form for Failure URL.
 */
public class SearchForm {

    /**
     * Default constructor for SearchForm.
     */
    public SearchForm() {
    }

    /**
     * The URL field for searching failure URLs.
     */
    public String url;

    /**
     * The minimum error count field for filtering failure URLs.
     */
    public String errorCountMin;

    /**

        List<String> urls = new ArrayList<>();
        urls.add("http://example.com/page1");
        urls.add("");
        urls.add("http://example.com/page2");
        urls.add("   ");
        urls.add("http://example.com/page3");

        Set<RequestData> result = crawlerThread.getAnchorSet(urls);
        assertNotNull(result, "getAnchorSet should filter out blank strings");

## OpenAPI Custom URL { #openapi-custom-url }

To customize the OpenAPI URL (or remove it), read the docs for [Tutorial - Metadata and Docs URLs](../tutorial/metadata.md#openapi-url).

## OpenAPI Docs URLs { #openapi-docs-urls }

    }

    /**
     * Stores a new failure URL or updates an existing one with error information.
     * Creates a new failure URL entry or increments the error count for an existing URL.
     *
     * @param crawlingConfig the crawling configuration associated with the failure
     * @param errorName the name/type of the error that occurred
     * @param url the URL that failed to be crawled

        } catch (Exception e) {
            fail("sendToSlack() should not throw an exception with blank URLs: " + e.getMessage());
        }
    }

    @Test
    public void test_sendToSlack_withNullWebhookUrls() {
        // Setup mock configuration with null webhook URLs
        ComponentUtil.setFessConfig(new MockFessConfig() {
            @Override
            public String getSlackWebhookUrls() {

    @Test
    public void test_constructor_withValidUrls() {
        // Test with normal URLs
        String parentUrl = "http://example.com/parent";
        String url = "http://example.com/child";
        ContentNotFoundException exception = new ContentNotFoundException(parentUrl, url);

        assertNotNull(exception);

===========

Although you provide only the URL, OkHttp plans its connection to your webserver using three types: URL, Address, and Route.

### [URLs](https://square.github.io/okhttp/5.x/okhttp/okhttp3/-http-url/)

URLs (like `https://github.com/square/okhttp`) are fundamental to HTTP and the Internet. In addition to being a universal, decentralized naming scheme for everything on the web, they also specify how to access web resources.

URLs are abstract:

    }

    /**
     * Adds a URL to the crawling queue.
     * @param url The URL to add.
     */
    public void addUrl(final String url) {
        try {
            urlQueueService.add(crawlerContext.sessionId, url);
            if (logger.isDebugEnabled()) {
                logger.debug("Added URL to queue: url={}, sessionId={}", url, crawlerContext.sessionId);
            }

private val TLS_12 =
  ConnectionSpec
    .Builder(ConnectionSpec.RESTRICTED_TLS)
    .tlsVersions(TlsVersion.TLS_1_2)
    .build()

private fun testClient(
  urls: List<String>,
  client: OkHttpClient,
) {
  try {
    for (url in urls) {
      sendRequest(client, url)
    }
  } finally {
    client.dispatcher.executorService.shutdownNow()
    client.connectionPool.evictAll()
  }
}