throw new IOException("Failed to access " + url, e);
            }
        }

        /**
         * Gets the object head response from the S3 service.
         * This method caches the response to avoid repeated calls.
         *
         * @return The object head response
         * @throws IOException If an error occurs
         */
        private HeadObjectResponse getHeadObject() throws IOException {

                    if (fessConfig.isSmbRoleFromFile() || fessConfig.isFileRoleFromFile() || fessConfig.isFtpRoleFromFile()) {
                        // head method
                        responseData =
                                client.execute(RequestDataBuilder.newRequestData().head().url(url).weight(urlQueue.getWeight()).build());
                        if (responseData == null) {
                            return true;

     *
     * @return the {@link UrlQueue} instance associated with the current thread, or {@code null} if none is set.
     */
    public static UrlQueue<?> getUrlQueue() {
        return URL_QUEUE_THREAD_LOCAL.get();
    }

    /**
     * Sets the URL queue for the current thread. If the provided URL queue is null,
     * the URL queue for the current thread is removed.
     *

        // start
        AccessTimeoutTarget accessTimeoutTarget = null;
        TimeoutTask accessTimeoutTask = null;
        if (accessTimeout != null) {
            accessTimeoutTarget = new AccessTimeoutTarget(Thread.currentThread());
            accessTimeoutTask = TimeoutManager.getInstance().addTimeoutTarget(accessTimeoutTarget, accessTimeout, false);
        }

        try {

    /**
     * Priority for crawler threads.
     */
    protected int threadPriority = Thread.NORM_PRIORITY;

    /**
     * Reference to the parent thread that started the crawler.
     */
    protected Thread parentThread;

    /**
     * The thread group for crawler threads.
     */
    protected ThreadGroup crawlerThreadGroup;

    /**

        } catch (final UnsupportedEncodingException e) {
            throw new CurlException("Invalid encoding: " + encoding, e);
        }
    }

    /**
     * Sets the thread pool for executing the request.
     *
     * @param threadPool the thread pool
     * @return this CurlRequest instance
     */
    public CurlRequest threadPool(final ForkJoinPool threadPool) {
        this.threadPool = threadPool;

        Set<String> valueSet = Collections.synchronizedSet(new HashSet<>());
        Thread[] threads = new Thread[threadNum];
        ESSourceReader reader = new ESSourceReader(client, settings, indexName);
        reader.setScrollSize(100);
        for (int i = 0; i < threadNum; i++) {
            threads[i] = new Thread(() -> {
                Map<String, Object> source;

        this.crawlingExecutionInterval = crawlingExecutionInterval;
    }

    /**
     * Sets the thread priority for crawler threads.
     * This priority will be applied to all newly created crawler threads.
     *
     * @param crawlerPriority thread priority (typically Thread.MIN_PRIORITY to Thread.MAX_PRIORITY)
     */
    public void setCrawlerPriority(final int crawlerPriority) {

 * <p>
 * <strong>Thread-Safety:</strong> This class is <strong>NOT thread-safe</strong>.
 * It extends {@link LinkedHashMap} without synchronization. If multiple threads access
 * an instance concurrently, and at least one thread modifies the map structurally,
 * it must be synchronized externally.
 * </p>
 * <p>
 * For thread-safe usage, wrap with {@link java.util.Collections#synchronizedMap(Map)}:

 * The delay is calculated based on the configured delayMillisBeforeProcessing parameter.
 * If the time since the last request to the host is less than the configured delay,
 * the thread waits until the delay has elapsed.
 * This class is thread-safe.
 * The cache automatically evicts entries after 1 hour of inactivity to prevent memory leaks.
 */
public class HostIntervalController extends DefaultIntervalController {