RequestData data = RequestDataBuilder.newRequestData().url(null).build();

        assertNull(data.getUrl());
    }

    public void test_realWorldUsageExample1() {
        // Real-world example: crawling a web page
        RequestData data = RequestDataBuilder.newRequestData().get().url("https://example.com/article/12345").weight(1.0f).build();

        assertNotNull(data);
        assertEquals(Method.GET, data.getMethod());

          echo "Error: Unable to autodetect the YJP library location. Please set YJPLIB variable" >&2
          exit 1
        fi
        MAVEN_OPTS="-agentpath:$YJPLIB=onexit=snapshot,onexit=memory,tracing,onlylocal $MAVEN_OPTS"
        ;;
      --enc)
        MAVEN_MAIN_CLASS="org.apache.maven.cling.MavenEncCling"
        ;;
      --shell)
        MAVEN_MAIN_CLASS="org.apache.maven.cling.MavenShellCling"

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author hayato
 *
 */
public class HostIntervalControllerTest extends PlainTestCase {

    /**
     * Test that crawling intervals for the same host work correctly.
     */
    public void test_delayBeforeProcessing() {
        // Number of concurrent tasks
        final int numTasks = 100;
        // Interval in milliseconds

         * than a handful of exceptions. But it seems prudent to set a cap on how many we'll cache.
         * This avoids out-of-control memory consumption, and it keeps the cache from growing so
         * large that doing the lookup is noticeably slower than redoing the work would be.
         *
         * Ideally we'd have a real eviction policy, but until we see a problem in practice, I hope

![Administrationsoberfläche](https://fess.codelibs.org/_images/fess_admin_dashboard.png)

Sie können in der Administrationsoberfläche (Web, Datei, Datenspeicher) Crawling-Ziele in den Crawler-Konfigurationsseiten registrieren und den Crawler manuell auf der [Scheduler-Seite](https://fess.codelibs.org/15.3/admin/scheduler-guide.html) starten.

## Migration von einem anderen Suchanbieter

2. **Browser header**: `Accept-Language` header
3. **Fallback**: English (from `fess_label.properties` and `fess_message.properties`)

### Document Language Detection

During crawling and indexing, Fess:

1. Detects language from document content using Apache Tika
2. Validates against `supported.languages` list
3. Creates language-specific fields (e.g., `content_ja`, `title_en`, `content_sv`)

        final String sessionId = "poll_session5";
        final int maxSize = 5;
        urlQueueService.setMaxCrawlingQueueSize(maxSize);

        // Insert more items than max crawling queue size
        final List<OpenSearchUrlQueue> urlQueueList = new ArrayList<>();
        for (int i = 1; i <= maxSize + 10; i++) {
            final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();

 * </ul>
 *
 * <p>References:</p>
 * <ul>
 * <li><a href="https://datatracker.ietf.org/doc/html/rfc9309">RFC 9309 - Robots Exclusion Protocol</a></li>
 * <li><a href="https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt">
 * Google's robots.txt Specification</a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

        echo Error: Unable to autodetect the YJP library location. Please set YJPLIB variable >&2
        exit /b 1
    )
    set "INTERNAL_MAVEN_OPTS=-agentpath:%YJPLIB%=onexit=snapshot,onexit=memory,tracing,onlylocal %INTERNAL_MAVEN_OPTS%"
) else if "%~1"=="--enc" (
    set "MAVEN_MAIN_CLASS=org.apache.maven.cling.MavenEncCling"
) else if "%~1"=="--shell" (
      set "MAVEN_MAIN_CLASS=org.apache.maven.cling.MavenShellCling"

import org.w3c.dom.Node;
import org.xml.sax.InputSource;

import jakarta.annotation.Resource;

/**
 * The {@code HtmlTransformer} class is responsible for transforming HTML responses
 * during the crawling process. It extracts data, identifies child URLs, and handles
 * character set encoding.
 * <p>
 * This class extends {@link AbstractTransformer} and utilizes various helper classes