<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <sitemap>
      <loc>http://www.example.com/sitemap1.xml.gz</loc>
      <lastmod>2004-10-01T18:23:17+00:00</lastmod>
   </sitemap>
   <sitemap>
      <loc>http://www.example.com/sitemap2.xml.gz</loc>
      <lastmod>2005-01-01</lastmod>
   </sitemap>

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml"/>
	<include path="crawler/client.xml"/>
	<include path="crawler/rule.xml"/>
	<include path="crawler/filter.xml"/>
	<include path="crawler/interval.xml"/>
	<include path="crawler/extractor.xml"/>
	<include path="crawler/mimetype.xml"/>

         * final String encoding = System.getProperty(SuggestConstants.USER_DICT_ENCODING); final Map<String, String>
         * args = new HashMap<>(); args.put("mode", "normal"); args.put("discardPunctuation", "false"); if
         * (Strings.isNullOrEmpty(path)) { args.put("userDictionary", path); } if (Strings.isNullOrEmpty(encoding)) {

                buf.append(value).append("\n");
            }
            return buf.toString();
        }

    }

    /**
     * Sets the encoding for the output.
     * @param outputEncoding The output encoding to set.
     */
    public void setOutputEncoding(final String outputEncoding) {
        this.outputEncoding = outputEncoding;
    }

    /**

     * @return the decoded text or empty string if decoding fails
     */
    protected String getDecodeText(final String value) {
        if (value == null) {
            return StringUtil.EMPTY;
        }
        try {
            return MimeUtility.decodeText(value);
        } catch (final UnsupportedEncodingException e) {
            logger.warn("Invalid encoding.", e);
            return StringUtil.EMPTY;
        }

     * @param stream the input stream to parse
     * @return the parsed RobotsTxt object, or null if disabled
     */
    public RobotsTxt parse(final InputStream stream) {
        return parse(stream, Constants.UTF_8);
    }

    /**
     * Parses a robots.txt file from the given input stream using the specified character encoding.
     * @param stream the input stream to parse

<?xml version="1.0" encoding="Shift_JIS"?>
<a>
<b>
<d>�^�C�g��</d>
</b>
<c>
�e�X�g
</c>

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="robotsTxtHelper" class="org.codelibs.fess.crawler.helper.RobotsTxtHelper"
		instance="prototype">
	</component>

     */
    public void setTempDir(final File tempDir) {
        this.tempDir = tempDir;
    }

    /**
     * Sets the output encoding for extracted text.
     *
     * @param outputEncoding the output encoding to set
     */
    public void setOutputEncoding(final String outputEncoding) {
        this.outputEncoding = outputEncoding;
    }

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />
	<include path="crawler/transformer.xml" />

	<component name="ruleManager"
		class="org.codelibs.fess.crawler.rule.impl.RuleManagerImpl" instance="prototype">
		<postConstruct name="addRule">
			<arg>sitemapsRule</arg>
		</postConstruct>