/** Current page number */
    private int currentPageNumber;

    /** Configuration ID */
    public String id;

    /** Configuration name */
    public String name;

    /** File system paths to crawl */
    public String paths;

    /** Number of threads for crawling */
    public String numOfThread;

    /** Interval time between crawling operations */
    public String intervalTime;

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="duplicateHost"/>
    </jsp:include>
    <main class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="relatedQuery"/>
    </jsp:include>
    <main class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

### Helpers

- **RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
- **SitemapsHelper**: Sitemap XML parsing, index handling
- **MimeTypeHelper**: MIME detection via Tika
- **EncodingHelper**: Charset detection with BOM
- **UrlConvertHelper**: URL normalization

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="labelType"/>
    </jsp:include>
    <main class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

import org.codelibs.fess.util.SystemUtil;

import jakarta.servlet.ServletContext;

/**
 * CrawlJob is responsible for executing the crawling process in Fess.
 * This job launches a separate crawler process that can crawl web sites, file systems,
 * and data sources based on the configured crawling settings.
 *
 * <p>The job supports selective crawling by specifying configuration IDs for different

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="webConfig"/>
    </jsp:include>
    <main class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">

  ~ See the License for the specific language governing permissions and
  ~ limitations under the License.
  -->
<!DOCTYPE module PUBLIC
        "-//Puppy Crawl//DTD Check Configuration 1.2//EN"
        "http://www.puppycrawl.com/dtds/configuration_1_2.dtd">
<module name="Checker">
    <module name="SuppressionFilter">
        <property name="file" value="${config_loc}/suppressions.xml"/>

     */
    protected ConcurrentHashMap<String, Pair<String, Pattern>> clientRuleCache = new ConcurrentHashMap<>();

    /**
     * Determines whether the content at the given URL has been updated since the last crawl.
     * This method implements incremental crawling by comparing timestamps and checking document
     * expiration. It also handles special cases for different URL schemes (SMB, file, FTP).
     *

<div class="wrapper">
    <jsp:include page="/WEB-INF/view/common/admin/header.jsp"></jsp:include>
    <jsp:include page="/WEB-INF/view/common/admin/sidebar.jsp">
        <jsp:param name="menuCategoryType" value="crawl"/>
        <jsp:param name="menuType" value="dataConfig"/>
    </jsp:include>
    <main class="content-wrapper">
        <div class="content-header">
            <div class="container-fluid">