preloadDate = new String(bytes, Constants.UTF_8);
            if (preloadDate.indexOf("<urlset") >= 0) {
                // XML Sitemaps
                bis.reset();
                return parseXmlSitemaps(bis);
            }
            if (preloadDate.indexOf("<sitemapindex") >= 0) {
                // XML Sitemaps Index
                bis.reset();
                return parseXmlSitemapsIndex(bis);
            }

     * @param c2 The second character.
     * @return The merged character.
     */
    public static char mergeChar(final char c1, final char c2) {
        if (c2 == 'ﾞ') {
            if ("ｶｷｸｹｺｻｼｽｾｿﾀﾁﾂﾃﾄﾊﾋﾌﾍﾎ".indexOf(c1) >= 0) {
                switch (c1) {
                case 'ｶ':
                    return 'ガ';
                case 'ｷ':
                    return 'ギ';
                case 'ｸ':

            } else {
                final StringBuilder buf = new StringBuilder(100);
                buf.append(tokenUrl);
                if (StringUtil.isNotBlank(tokenReqParams)) {
                    if (tokenUrl.indexOf('?') >= 0) {
                        buf.append('&');
                    } else {
                        buf.append('?');
                    }
                    buf.append(tokenReqParams);
                }

|typeof a.getAttribute!==C&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=fb.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.par...

     *
     * @param host the host string to decode
     * @return the decoded host string
     */
    protected String decode(final String host) {
        if (host.indexOf('%') == -1) {
            return host;
        }
        try {
            return URLDecoder.decode(host, encoding);
        } catch (final UnsupportedEncodingException e) {
            return host;
        }

            while (stream.incrementToken()) {
                final CharTermAttribute att = stream.getAttribute(CharTermAttribute.class);
                final String term = att.toString();
                final int pos = inputStr.indexOf(term, offset);
                if (pos > 0) {
                    final String tmp = inputStr.substring(offset, pos);
                    kanaBuf.append(transliterator.transliterate(tmp));
                    offset = pos;

     */
    boolean matches(final String path) {
        if (StringUtil.isBlank(path)) {
            return false;
        }

        try {
            final int pos = path.indexOf('/', 6);
            final URI uri = new URI(pos == -1 ? path : path.substring(0, pos));
            if (!"ftp".equals(uri.getScheme()) || (StringUtil.isNotBlank(server) && !server.equals(uri.getHost()))) {

        String url = u.trim();

        int idx = url.indexOf('#');
        if (idx >= 0) {
            url = url.substring(0, idx);
        }

        url = url.replace("/./", "/");

        idx = url.indexOf(";jsessionid");
        if (idx >= 0) {
            url = url.replaceFirst(";jsessionid=[a-zA-Z0-9\\.]*", "");
        }

        if (url.indexOf(' ') >= 0) {
            url = url.replace(" ", "%20");

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

                        reading = term;
                    }
                    reading = transliterator.transliterate(reading);

                    final int pos = text.indexOf(term, offset);
                    if (pos > 0) {
                        final String tmp = text.substring(offset, pos);
                        readingBuf.append(transliterator.transliterate(tmp));