httpClientPropertyMap.put(name, value);
        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

package org.codelibs.fess.crawler.exception;

/**
 * MimeTypeException is a custom exception class that extends CrawlerSystemException.
 * It is used to indicate exceptions related to MIME type handling during the crawling process.
 * This exception can be thrown with a message, a cause, or both.
 */
public class MimeTypeException extends CrawlerSystemException {

    private static final long serialVersionUID = 1L;

    /**

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * An exception indicating that the execution of a process has timed out.
 * This exception extends {@link org.codelibs.fess.crawler.exception.ExtractException}.
 *
 */
public class ExecutionTimeoutException extends ExtractException {

     */
    @Override
    public ResponseData doGet(final String uri) {
        return processRequest(uri, true);
    }

    /**
     * Processes a request for the given URI.
     *
     * @param uri the URI to process
     * @param includeContent whether to include content in the response
     * @return the response data
     * @throws CrawlingAccessException if the request fails
     */

```

### Monitoring

```java
// Monitor crawling progress
while (crawler.crawlerContext.getStatus() == CrawlerStatus.RUNNING) {
    int processed = dataService.getCount(sessionId);
    System.out.println("Processed: " + processed + " URLs");
    Thread.sleep(5000);
}
```

## Contributing

We welcome contributions to Fess Crawler! Please follow these guidelines:

 * This class provides methods to set and get various parameters related to the crawling process.
 *
 * <p>This class is final and cannot be instantiated.</p>
 *
 * <p>The following parameters are managed:</p>
 * <ul>
 *   <li>{@link UrlQueue} - The queue of URLs to be crawled.</li>
 *   <li>{@link CrawlerContext} - The context of the current crawling process.</li>
 *   <li>{@link UrlQueueService} - The service for managing the URL queue.</li>

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * Exception thrown during the extraction process in the crawler.
 * This exception indicates a failure or error that occurred while extracting content from a crawled resource.
 * It extends {@link org.codelibs.fess.crawler.exception.CrawlerSystemException} and provides constructors

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.entity;

/**
 * The UrlQueue interface represents a queue of URLs to be processed by a web crawler.
 * It provides methods to get and set various properties of a URL queue entry.
 *
 * @param <IDTYPE> the type of the identifier for the URL queue entry
 */
public interface UrlQueue<IDTYPE> {

 */
package org.codelibs.fess.crawler.exception;

/**
 * CrawlingAccessException is an exception class that represents an issue encountered while accessing a resource during the crawling process.
 * It extends CrawlerSystemException and provides functionality to set and check the log level for the exception.
 *
 * <p>

    /**
     * Current status of the crawler.
     */
    protected volatile CrawlerStatus status = CrawlerStatus.INITIALIZING;

    /**
     * Filter for URLs to control which URLs are processed.
     */
    protected UrlFilter urlFilter;

    /**
     * Manager for crawling rules and configurations.
     */
    protected RuleManager ruleManager;

    /**