import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

                webFsCrawlerThread = new Thread((Runnable) () -> {
                    // crawl web
                    writeTimeToSessionInfo(crawlingInfoHelper, Constants.WEB_FS_CRAWLER_START_TIME);
                    webFsIndexHelper.crawl(options.sessionId, webConfigIdList, fileConfigIdList);
                    writeTimeToSessionInfo(crawlingInfoHelper, Constants.WEB_FS_CRAWLER_END_TIME);

    protected int crawlerPriority = Thread.NORM_PRIORITY;

    protected final List<Crawler> crawlerList = Collections.synchronizedList(new ArrayList<>());

    public void crawl(final String sessionId, final List<String> webConfigIdList, final List<String> fileConfigIdList) {
        final boolean runAll = webConfigIdList == null && fileConfigIdList == null;
        final List<WebConfig> webConfigList;

Fess also contains a Crawler, which can crawl documents on a [web server](https://fess.codelibs.org/14.17/admin/webconfig-guide.html), [file system](https://fess.codelibs.org/14.17/admin/fileconfig-guide.html), or [Data Store](https://fess.codelibs.org/14.17/admin/dataconfig-guide.html) (such...

 *     toCrawl = readStartingUris();
 *   }
 *
 *   protected void runOneIteration() throws Exception {
 *     Uri uri = toCrawl.remove();
 *     Collection<Uri> newUris = crawl(uri);
 *     visited.add(uri);
 *     for (Uri newUri : newUris) {
 *       if (!visited.contains(newUri)) { toCrawl.add(newUri); }
 *     }
 *   }
 *
 *   protected void shutDown() throws Exception {