* Provides RESTful API endpoints for managing file authentication settings in the Fess search engine.
 * File authentication settings define access credentials and permissions for file-based crawling.
 *
 */
public class ApiAdminFileauthAction extends FessApiAdminAction {

    private static final Logger logger = LogManager.getLogger(ApiAdminFileauthAction.class);

    //                                                                           Attribute
    //                                                                           =========
    /** Helper for crawling configuration management. */
    @Resource
    protected CrawlingConfigHelper crawlingConfigHelper;

    // ===================================================================================

        RequestData data = RequestDataBuilder.newRequestData().url(null).build();

        assertNull(data.getUrl());
    }

    public void test_realWorldUsageExample1() {
        // Real-world example: crawling a web page
        RequestData data = RequestDataBuilder.newRequestData().get().url("https://example.com/article/12345").weight(1.0f).build();

        assertNotNull(data);
        assertEquals(Method.GET, data.getMethod());

import org.dbflute.utflute.core.PlainTestCase;

/**
 * @author hayato
 *
 */
public class HostIntervalControllerTest extends PlainTestCase {

    /**
     * Test that crawling intervals for the same host work correctly.
     */
    public void test_delayBeforeProcessing() {
        // Number of concurrent tasks
        final int numTasks = 100;
        // Interval in milliseconds

2. **Browser header**: `Accept-Language` header
3. **Fallback**: English (from `fess_label.properties` and `fess_message.properties`)

### Document Language Detection

During crawling and indexing, Fess:

1. Detects language from document content using Apache Tika
2. Validates against `supported.languages` list
3. Creates language-specific fields (e.g., `content_ja`, `title_en`, `content_sv`)

        final String sessionId = "poll_session5";
        final int maxSize = 5;
        urlQueueService.setMaxCrawlingQueueSize(maxSize);

        // Insert more items than max crawling queue size
        final List<OpenSearchUrlQueue> urlQueueList = new ArrayList<>();
        for (int i = 1; i <= maxSize + 10; i++) {
            final OpenSearchUrlQueue urlQueue = new OpenSearchUrlQueue();

 * </ul>
 *
 * <p>References:</p>
 * <ul>
 * <li><a href="https://datatracker.ietf.org/doc/html/rfc9309">RFC 9309 - Robots Exclusion Protocol</a></li>
 * <li><a href="https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt">
 * Google's robots.txt Specification</a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

import org.w3c.dom.Node;
import org.xml.sax.InputSource;

import jakarta.annotation.Resource;

/**
 * The {@code HtmlTransformer} class is responsible for transforming HTML responses
 * during the crawling process. It extracts data, identifies child URLs, and handles
 * character set encoding.
 * <p>
 * This class extends {@link AbstractTransformer} and utilizes various helper classes

        transformer.addTransformationRule("<[^>]+>", ""); // Remove HTML tags
        transformer.addTransformationRule("\\s+", " "); // Normalize whitespace

        // Simulate crawling response
        ResponseData responseData = new ResponseData();
        responseData.setUrl("http://example.com/page.html");
        responseData.setParentUrl("http://example.com/");

        }
        return false;
    }

    /**
     * Create the parameters for extraction.
     * @param responseData The response data.
     * @param crawlingConfig The crawling configuration.
     * @return The parameters for extraction.
     */
    protected Map<String, String> createExtractParams(final ResponseData responseData, final CrawlingConfig crawlingConfig) {