* @return The index settings as a string.
     * @throws IOException If an I/O error occurs.
     */
    protected String loadIndexSettings() throws IOException {
        final String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
        final StringBuilder sb = new StringBuilder();
        try (BufferedReader br =

    /**
     * Load index settings.
     * @return Index settings
     * @throws IOException I/O exception
     */
    protected String loadIndexSettings() throws IOException {
        final String dictionaryPath = System.getProperty("fess.dictionary.path", StringUtil.EMPTY);
        final StringBuilder sb = new StringBuilder();
        try (BufferedReader br = new BufferedReader(new InputStreamReader(

     * @return true if the crawling should continue, false otherwise.
     */
    protected boolean isContinue(final int tcCount) {
        if (!crawlerContainer.available()) {
            // system shutdown
            return false;
        }

        boolean isContinue = false;
        if (tcCount < crawlerContext.maxThreadCheckCount) {

        if (roles == null || roles.length == 0) {
            this.roles = new String[] { SuggestConstants.DEFAULT_ROLE };
        } else {
            this.roles = new String[roles.length];
            System.arraycopy(roles, 0, this.roles, 0, roles.length);
        }

        this.languages = languages != null ? languages : new String[] {};

        kinds = new Kind[] { kind };
        if (userBoost > 1) {

        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

        <match value="allow:" type="stringignorecase" offset="0"/>
        <match value="disallow:" type="stringignorecase" offset="0"/>
        <match value="sitemap:" type="stringignorecase" offset="0"/>
        <match value="\nuser-agent:" type="stringignorecase" offset="0:1000"/>
        <match value="\nallow:" type="stringignorecase" offset="0:1000"/>

 * instances to improve performance by reusing connections.
 * </p>
 *
 * <p>
 * The client can be configured with FTP-specific settings via init parameters, such as:
 * </p>
 * <ul>
 *   <li>ftpConfigSystemKey: The system key used to configure the FTPClientConfig.</li>
 *   <li>ftpConfigServerLanguageCode: The language code used by the FTP server.</li>
 *   <li>ftpConfigServerTimeZoneId: The time zone ID of the FTP server.</li>