}
        return false;
    }

    /**
     * Generates SitemapSet instance.
     *
     * This method does not close the input stream.
     *
     * @param in Input stream for a sitemap
     * @return a sitemap set
     */
    public SitemapSet parse(final InputStream in) {
        return parse(in, true);
    }

    protected SitemapSet parse(final InputStream in, final boolean recursive) {

                    rule.setResponseProcessor(container.getComponent("sitemapsResponseProcessor"));
                    rule.setRuleId("sitemapsRule");
                    rule.addRule("url", ".*sitemap.*");
                })//
                .<DefaultResponseProcessor> singleton("defaultResponseProcessor", DefaultResponseProcessor.class, processor -> {

 * the client is Microsoft Internet Explorer.
 */

public class NetworkExplorer extends HttpServlet {

    private static LogStream log = LogStream.getInstance();

    private MimeMap mimeMap;
    private String style;
    private NtlmSsp ntlmSsp;
    private boolean credentialsSupplied;
    private boolean enableBasic;
    private boolean insecureBasic;
    private String realm, defaultDomain;

    }

    protected void addSitemapsFromRobotsTxt(final UrlQueue<?> urlQueue) {
        final String[] sitemaps = crawlerContext.removeSitemaps();
        if (sitemaps != null) {
            for (final String childUrl : sitemaps) {
                try {
                    storeChildUrl(childUrl, urlQueue.getUrl(), null, urlQueue.getDepth() == null ? 1 : urlQueue.getDepth() + 1);

							if i > 1 {
								file = fmt.Sprintf("%s/part.%d", file, i)
								filemap[file] = make(map[string]string)
								filemap[file][verID] = fmt.Sprintf("%s/part.%d/shard-%02d-of-%02d", verID, i, idx, ei.V2Obj.EcN+ei.V2Obj.EcM)
								filemap[file][verID+".json"] = buf.String()
							}
							partDataToVerID[dataFile] = [2]string{file, verID}
						}

            file.delete();
            file.mkdirs();
            file.deleteOnExit();
            fileTransformer.setPath(file.getAbsolutePath());
            crawler.addUrl(url + "sitemaps.xml");
            crawler.crawlerContext.setMaxAccessCount(maxCount);
            crawler.crawlerContext.setNumOfThread(numOfThread);
            crawler.urlFilter.addInclude(url + ".*");

                    if (robotsTxt != null) {
                        final String[] sitemaps = robotsTxt.getSitemaps();
                        if (sitemaps.length > 0) {
                            crawlerContext.addSitemaps(sitemaps);
                        }

                        final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent);

   */
  @GwtIncompatible // java.io.ObjectOutputStream
  @J2ktIncompatible
  private void writeObject(ObjectOutputStream stream) throws IOException {
    stream.defaultWriteObject();
    Serialization.writeMap(this, stream);
  }

  @GwtIncompatible // java.io.ObjectInputStream
  @J2ktIncompatible
  private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException {

   */
  @GwtIncompatible // java.io.ObjectOutputStream
  @J2ktIncompatible
  private void writeObject(ObjectOutputStream stream) throws IOException {
    stream.defaultWriteObject();
    Serialization.writeMap(this, stream);
  }

  @GwtIncompatible // java.io.ObjectInputStream
  @J2ktIncompatible
  private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException {

   *
   * @since 20.0
   */
  public static final MediaType VTT_UTF_8 = createConstantUtf8(TEXT_TYPE, "vtt");

  /* image types */
  /**
   * <a href="https://en.wikipedia.org/wiki/BMP_file_format">Bitmap file format</a> ({@code bmp}
   * files).
   *
   * @since 13.0
   */
  public static final MediaType BMP = createConstant(IMAGE_TYPE, "bmp");

  /**