}

    /**
     * Gets the crawl delay value for the specified user agent from robots.txt.
     * The crawl delay specifies the time (in seconds) to wait between successive requests.
     *
     * @param userAgent The user agent string to match against robots.txt directives
     * @return The crawl delay value in seconds. Returns 0 if no matching directive is found
     *         or no crawl delay is specified for the matching directive.
     */

     */
    protected String path;

    /**
     * A string to replace ?.
     */
    protected String questionStr = "_QUEST_";

    /**
     * A string to replace :.
     */
    protected String colonStr = "_CLN_";

    /**
     * A string to replace ;.
     */
    protected String semicolonStr = "_SCLN_";

    /**
     * A string to replace &.
     */
    protected String ampersandStr = "_AMP_";

        final File targetFile = ResourceUtil.getResourceAsFile("test");
        String path = targetFile.getAbsolutePath();
        if (!path.startsWith("/")) {
            path = "/" + path.replace('\\', '/');
        }
        final String url = "file:" + path;

        final int maxCount = 3;
        final int numOfThread = 2;

        final File file = File.createTempFile("crawler-", "");

            responseData.setContentLength(0);
            ftpClientQueue.offer(client);
        }
    }

    /**
     * Validates the FTP request by checking the reply code from the FTP server.
     * If the reply code indicates an error, a CrawlingAccessException is thrown.
     *
     * @param client the FTPClient instance to validate
     * @throws CrawlingAccessException if the FTP request failed
     */

            final String text = source.get(FieldNames.TEXT).toString();
            if (skipDuplicateWords) {
                final String duplicateCheckStr = text.replace(" ", "");
                if (words.stream().map(word -> word.replace(" ", "")).anyMatch(word -> word.equals(duplicateCheckStr))) {
                    // skip duplicate word.
                    continue;
                }
            }

     * @param credentials The credentials.
     * @param value The value to replace.
     * @return The value with credentials replaced.
     */
    protected String replaceCredentials(final Credentials credentials, final String value) {
        if (StringUtil.isNotBlank(value)) {
            return value.replace(USERNAME, credentials.getUserPrincipal().getName()).replace(PASSWORD, credentials.getPassword());
        }

                        crawlerContext.intervalController.delay(IntervalController.NO_URL_IN_QUEUE);
                    }

                    threadCheckCount++;
                }

                // interval
                if (crawlerContext.intervalController != null) {
                    crawlerContext.intervalController.delay(IntervalController.WAIT_NEW_URL);
                }
            }

        final File targetFile = ResourceUtil.getResourceAsFile("test");
        String path = targetFile.getAbsolutePath();
        if (!path.startsWith("/")) {
            path = "/" + path.replace('\\', '/');
        }
        final String url = "file:" + path;

        final int maxCount = 3;
        final int numOfThread = 2;

        final File file = File.createTempFile("crawler-", "");

        } catch (final Exception e) {
            if (logger.isDebugEnabled()) {
                logger.debug("Failed to replace with new index.", e);
            }
            throw new SuggesterException("Failed to replace with new index.", e);
        }
    }

    /**
     * Switches the search alias to the current update index.
     */
    public void switchIndex() {

            putValue(data, "Message-Number", message.getMessageNumber());
            putValue(data, "Received-Date", getReceivedDate(message));
            putValue(data, "Reply-To", message.getReplyTo());
            putValue(data, "Sender", message.getSender());
            putValue(data, "Sent-Date", message.getSentDate());
            putValue(data, "Size", message.getSize());