final int pos = text.indexOf(term, offset);
                    if (pos > 0) {
                        final String tmp = text.substring(offset, pos);
                        readingBuf.append(transliterator.transliterate(tmp));
                        offset = pos;
                    } else if (pos == -1) {
                        continue;
                    }

                final String term = att.toString();
                final int pos = inputStr.indexOf(term, offset);
                if (pos > 0) {
                    final String tmp = inputStr.substring(offset, pos);
                    kanaBuf.append(transliterator.transliterate(tmp));
                    offset = pos;
                } else if (pos == -1) {
                    continue;
                }

				"application/vnd.oma.drm.risd+xml",
				"application/vnd.oma.group-usage-list+xml",
				"application/vnd.oma.poc.detailed-progress-report+xml",
				"application/vnd.oma.poc.final-report+xml",
				"application/vnd.oma.poc.groups+xml",
				"application/vnd.oma.poc.invocation-descriptor+xml",
				"application/vnd.oma.poc.optimized-progress-report+xml",
				"application/vnd.oma.xcap-directory+xml",
				"application/vnd.omads-email+xml",

    boolean matches(final String path) {
        if (StringUtil.isBlank(path)) {
            return false;
        }

        try {
            final int pos = path.indexOf('/', 6);
            final URI uri = new URI(pos == -1 ? path : path.substring(0, pos));
            if (!"ftp".equals(uri.getScheme()) || (StringUtil.isNotBlank(server) && !server.equals(uri.getHost()))) {
                return false;
            }

                        final int pos = s.indexOf('=');
                        if (pos == -1 || pos == s.length() - 1) {
                            name = s;
                            value = StringUtil.EMPTY;
                        } else {
                            name = s.substring(0, pos);
                            value = s.substring(pos + 1);
                        }

     * @param regex The regular expression for the client.
     * @param client The CrawlerClient instance.
     * @param pos The position to add the client.
     */
    @Override
    public void addClient(final String regex, final CrawlerClient client, final int pos) {
        factory.addClient(regex, client, pos);
    }

    /**
     * Returns the hash code of the wrapped factory.
     * @return The hash code.
     */

     * @param regex The regular expression to match URLs.
     * @param client The CrawlerClient instance.
     * @param pos The position to add the client.
     */
    public void addClient(final String regex, final CrawlerClient client, final int pos) {
        if (StringUtil.isBlank(regex)) {
            throw new CrawlerSystemException("A regular expression is null.");
        }

     * @return the file name
     */
    private String getFileName(final String resourceName) {
        final String name = resourceName.replaceAll("/+$", "");
        final int pos = name.lastIndexOf('/');
        if (pos >= 0) {
            return name.substring(pos + 1);
        }
        return name;
    }

    /**
     * Sets the office manager for document conversion.
     *

                    if (doc == null || docs.size() >= docPerReq) {
                        final SuggestIndexResponse res = indexFromDocument(docs.toArray(new Map[docs.size()]));
                        errors.addAll(res.getErrors());
                        numberOfSuggestDocs += res.getNumberOfSuggestDocs();
                        numberOfInputDocs += res.getNumberOfInputDocs();

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */