/** The key of the configuration. e.g. 100 */
    String PAGE_CRAWLING_INFO_PARAM_MAX_FETCH_SIZE = "page.crawling.info.param.max.fetch.size";

    /** The key of the configuration. e.g. 1000 */
    String PAGE_CRAWLING_INFO_MAX_FETCH_SIZE = "page.crawling.info.max.fetch.size";

    /** The key of the configuration. e.g. 100 */

        RequestData data = RequestDataBuilder.newRequestData().url(null).build();

        assertNull(data.getUrl());
    }

    public void test_realWorldUsageExample1() {
        // Real-world example: crawling a web page
        RequestData data = RequestDataBuilder.newRequestData().get().url("https://example.com/article/12345").weight(1.0f).build();

        assertNotNull(data);
        assertEquals(Method.GET, data.getMethod());

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author hayato
 *
 */
public class HostIntervalControllerTest extends PlainTestCase {

    /**
     * Test that crawling intervals for the same host work correctly.
     */
    public void test_delayBeforeProcessing() {
        // Number of concurrent tasks
        final int numTasks = 100;
        // Interval in milliseconds

![Administrationsoberfläche](https://fess.codelibs.org/_images/fess_admin_dashboard.png)

Sie können in der Administrationsoberfläche (Web, Datei, Datenspeicher) Crawling-Ziele in den Crawler-Konfigurationsseiten registrieren und den Crawler manuell auf der [Scheduler-Seite](https://fess.codelibs.org/15.3/admin/scheduler-guide.html) starten.

## Migration von einem anderen Suchanbieter

2. **Browser header**: `Accept-Language` header
3. **Fallback**: English (from `fess_label.properties` and `fess_message.properties`)

### Document Language Detection

During crawling and indexing, Fess:

1. Detects language from document content using Apache Tika
2. Validates against `supported.languages` list
3. Creates language-specific fields (e.g., `content_ja`, `title_en`, `content_sv`)

        final String sessionId = "poll_session5";
        final int maxSize = 5;
        urlQueueService.setMaxCrawlingQueueSize(maxSize);

        // Insert more items than max crawling queue size
        final List<OpenSearchUrlQueue> urlQueueList = new ArrayList<>();
        for (int i = 1; i <= maxSize + 10; i++) {
            final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();

 * </ul>
 *
 * <p>References:</p>
 * <ul>
 * <li><a href="https://datatracker.ietf.org/doc/html/rfc9309">RFC 9309 - Robots Exclusion Protocol</a></li>
 * <li><a href="https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt">
 * Google's robots.txt Specification</a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

import org.w3c.dom.Node;
import org.xml.sax.InputSource;

import jakarta.annotation.Resource;

/**
 * The {@code HtmlTransformer} class is responsible for transforming HTML responses
 * during the crawling process. It extracts data, identifies child URLs, and handles
 * character set encoding.
 * <p>
 * This class extends {@link AbstractTransformer} and utilizes various helper classes

     * Test if a method is a prime declaration or an overrides that change the signature.
     *
     * There's no way to tell from the byte code that a method overrides the signature
     * of a parent declaration other than crawling up the type hierarchy.
     */
    private
    fun isSignificantDeclaration(methodNode: MethodNode): Boolean {

        if (methodNode.access.isSynthetic) return false

        if (!hasSuperType) return true

        transformer.addTransformationRule("<[^>]+>", ""); // Remove HTML tags
        transformer.addTransformationRule("\\s+", " "); // Normalize whitespace

        // Simulate crawling response
        ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/page.html");
        responseData.setParentUrl("http://example.com/");