if (doc == null || docs.size() >= docPerReq) {
                        final SuggestIndexResponse res = indexFromDocument(docs.toArray(new Map[docs.size()]));
                        errors.addAll(res.getErrors());
                        numberOfSuggestDocs += res.getNumberOfSuggestDocs();
                        numberOfInputDocs += res.getNumberOfInputDocs();

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */

    <root-XML localName="map" namespaceURI="http://docs.oasis-open.org/namespace"/>
    <glob pattern="*.ditamap"/>
  </mime-type>
  <mime-type type="application/dita+xml;format=topic">
    <sub-class-of type="application/dita+xml"/>
    <_comment>DITA Topic</_comment>
    <root-XML localName="topic"/>
    <root-XML localName="topic" namespaceURI="http://docs.oasis-open.org/namespace"/>