logger.debug("Crawling access exception at url: {}", urlQueue.getUrl(), e);
        } else if (e.isInfoEnabled()) {
            logger.info(e.getMessage());
        } else if (e.isWarnEnabled()) {
            logger.warn("Crawling access exception at url: {}", urlQueue.getUrl(), e);
        } else if (e.isErrorEnabled()) {
            logger.error("Crawling access exception at url: {}", urlQueue.getUrl(), e);

import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * The Crawler class is the main class for web crawling. It manages the crawling process,
 * including adding URLs to the queue, filtering URLs, managing crawler threads,
 * and handling the overall crawling lifecycle.
 *
 * <p>It implements the Runnable interface to be executed in a separate thread,

/**
 * Helper class for managing data crawling operations in Fess.
 * This class coordinates the execution of data store crawling processes,
 * managing multiple concurrent crawling threads and handling the indexing
 * of crawled documents into the search engine.
 *
 * <p>The DataIndexHelper supports:</p>
 * <ul>
 *   <li>Concurrent crawling of multiple data configurations</li>

labels.createdTime=Created Time
labels.depth=Depth
labels.excludedPaths=Excluded Paths for Crawling
labels.excludedUrls=Excluded URLs for Crawling
labels.excludedDocPaths=Excluded Paths for Searching
labels.excludedDocUrls=Excluded URLs for Searching
labels.hostname=Hostname
labels.id=ID
labels.includedPaths=Included Paths for Crawling
labels.includedUrls=Included URLs for Crawling
labels.includedDocPaths=Included Paths for Searching

    @Size(max = 10)
    public String thumbnail;

    /**
     * Types of crawling failures to ignore during crawling operations.
     * Specified failure types will not be logged or counted as errors.
     */
    @Size(max = 1000)
    public String ignoreFailureType;

    /**
     * Threshold for failure count before stopping crawling of a URL.
     * Set to -1 to disable the threshold check.
     */
    @Required

    /**
     * The namespace identifier for the crawling session.
     * Used to organize and identify crawling activities in the system.
     * Defaults to the system crawling info name.
     */
    protected String namespace = Constants.CRAWLING_INFO_SYSTEM_NAME;

    /**
     * Array of web crawling configuration IDs to process.

 * <ul>
 * <li>Web crawling - crawls web sites and web content</li>
 * <li>File system crawling - crawls file systems and documents</li>
 * <li>Data store crawling - crawls databases and other data sources</li>
 * <li>Combined crawling - runs multiple crawling types simultaneously</li>
 * </ul>
 *
 * <p>Command line usage:
 * <pre>
 * java org.codelibs.fess.exec.Crawler [options...]

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;

import jakarta.annotation.PostConstruct;

/**
 * Helper class for managing crawling configurations.
 * Provides functionality to store, retrieve, and manage different types of crawling configurations
 * including web, file, and data configurations. Supports caching and session-based configuration management.
 */
public class CrawlingConfigHelper {

- Graceful degradation (e.g., robots.txt parsing continues on errors)

---

## Key Components

### Crawler (`Crawler.java`)

Main orchestrator for crawling operations.

**Key Methods**:
```java
String execute()                // Start crawling, return session ID
void addUrl(String url)         // Add URL to queue
void cleanup(String sessionId)  // Clean up session
void stop()                     // Stop gracefully

        return redirectWith(getClass(), moreUrl("crawlingConfigForm"));
    }

    /**
     * Creates a crawling configuration and proceeds to the start crawling form.
     *
     * @param form the form containing crawling configuration data
     * @return HTML response redirecting to the start crawling form
     */
    @Execute
    @Secured({ ROLE })