public void setInitialBufferSize(final int initialBufferSize) {
        this.initialBufferSize = initialBufferSize;
    }

    /**
     * Sets whether duplicated terms are replaced.
     * @param replaceDuplication If true, duplicated terms are replaced.
     */
    public void setReplaceDuplication(final boolean replaceDuplication) {
        this.replaceDuplication = replaceDuplication;
    }

    /**

     * where '*' is replaced with '.*' for pattern matching, and stored case-insensitively.
     *
     * @param directive The directive to add to the robots.txt rules
     */
    public void addDirective(final Directive directive) {
        directiveMap.put(Pattern.compile(directive.getUserAgent().replace("*", ".*"), Pattern.CASE_INSENSITIVE), directive);
    }

    /**

    }

    /**
     * Replaces credentials in the given value.
     * @param credentials The credentials.
     * @param value The value to replace.
     * @return The value with credentials replaced.
     */
    protected String replaceCredentials(final Credentials credentials, final String value) {
        if (StringUtil.isNotBlank(value)) {

        return stripInvalidXMLCharacters(//
                value//
                        .replace("&", "&") //
                        .replace("<", "&lt;")//
                        .replace(">", "&gt;")//
                        .replace("\"", "&quot;")//
                        .replace("\'", "&apos;")//
        );
    }

    /**
     * Strips invalid XML characters from the input string.

        } catch (final Exception e) {
            if (logger.isDebugEnabled()) {
                logger.debug("Failed to replace with new index.", e);
            }
            throw new SuggesterException("Failed to replace with new index.", e);
        }
    }

    /**
     * Switches the search alias to the current update index.
     */
    public void switchIndex() {

     */
    protected String path;

    /**
     * A string to replace ?.
     */
    protected String questionStr = "_QUEST_";

    /**
     * A string to replace :.
     */
    protected String colonStr = "_CLN_";

    /**
     * A string to replace ;.
     */
    protected String semicolonStr = "_SCLN_";

    /**
     * A string to replace &.
     */
    protected String ampersandStr = "_AMP_";

    }

    public void test_doGet_dir() {
        final File file = ResourceUtil.getResourceAsFile("test");
        String path = file.getAbsolutePath();
        if (!path.startsWith("/")) {
            path = "/" + path.replace('\\', '/');
        }
        try {
            fsClient.doGet("file://" + path);
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();

        String input = content.replaceAll("[\\r\\n]", " ");
        if (ignoreCommentTag) {
            input = input.replaceAll("<!--[^>]+-->", "");
        } else {
            input = input.replace("<!--", "").replace("-->", "");
        }
        final Matcher matcher = getTagPattern().matcher(input);
        final StringBuffer sb = new StringBuffer();
        final Pattern attrPattern = Pattern.compile("\\s[^ ]+=\"([^\"]*)\"");

        final File targetFile = ResourceUtil.getResourceAsFile("test");
        String path = targetFile.getAbsolutePath();
        if (!path.startsWith("/")) {
            path = "/" + path.replace('\\', '/');
        }
        final String url = "file:" + path;

        final int maxCount = 3;
        final int numOfThread = 2;

        final File file = File.createTempFile("crawler-", "");

     * @return the escaped query string
     */
    public static String escapeWildcardQuery(final String query) {
        return query.replace("*", "\\*").replace("?", "\\?");
    }