<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="labelType"/>
    </jsp:include>
    <div class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

import org.codelibs.fess.util.SystemUtil;

import jakarta.servlet.ServletContext;

/**
 * CrawlJob is responsible for executing the crawling process in Fess.
 * This job launches a separate crawler process that can crawl web sites, file systems,
 * and data sources based on the configured crawling settings.
 *
 * <p>The job supports selective crawling by specifying configuration IDs for different

     */
    protected ConcurrentHashMap<String, Pair<String, Pattern>> clientRuleCache = new ConcurrentHashMap<>();

    /**
     * Determines whether the content at the given URL has been updated since the last crawl.
     * This method implements incremental crawling by comparing timestamps and checking document
     * expiration. It also handles special cases for different URL schemes (SMB, file, FTP).
     *

import org.codelibs.fess.crawler.helper.ContentLengthHelper;
import org.codelibs.fess.crawler.helper.MimeTypeHelper;

import jakarta.annotation.Resource;

/**
 * FileSystemClient is CrawlerClient implementation to crawl files on a file
 * system.
 *
 * @author shinsuke
 *
 */
public class FileSystemClient extends AbstractCrawlerClient {

    /** Logger instance for this class */

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="fileConfig"/>
    </jsp:include>
    <div class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="webConfig"/>
    </jsp:include>
    <div class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

    @Resource
    private RequestHeaderService requestHeaderService;
    @Resource
    private ReqHeaderPager reqHeaderPager;
    /** Web configuration service for managing web crawl configurations. */
    @Resource
    protected WebConfigService webConfigService;

    // ===================================================================================

        try {
            crawlJob.execute();
            fail("Should throw JobProcessingException");
        } catch (JobProcessingException e) {
            assertTrue(e.getMessage().contains("Failed to execute a crawl job"));
            assertNotNull(e.getCause());
            assertEquals("Test exception", e.getCause().getMessage());
        }
    }

    // Test execute with JobProcessingException in executeCrawler

     */
    protected CrawlerClient getClient(final String url) {
        return clientFactory.getClient(url);
    }

    /**
     * Checks if the content has been updated since the last crawl.
     * @param client The crawler client.
     * @param urlQueue The URL queue entry.
     * @return true if content is updated, false otherwise.
     */

 *     toCrawl = readStartingUris();
 *   }
 *
 *   protected void runOneIteration() throws Exception {
 *     Uri uri = toCrawl.remove();
 *     Collection<Uri> newUris = crawl(uri);
 *     visited.add(uri);
 *     for (Uri newUri : newUris) {
 *       if (!visited.contains(newUri)) { toCrawl.add(newUri); }
 *     }
 *   }
 *
 *   protected void shutDown() throws Exception {