buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');
            buf.append("<loc>http://localhost:7070/index.html</loc>").append('\n');

            buf = new StringBuilder();
            buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").append('\n');
            buf.append("<urlset ").append("xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">").append('\n');
            buf.append("<url>").append('\n');
            buf.append("<loc>http://localhost:7070/index.html</loc>").append('\n');

extractorFactory.addExtractor("text/html", tikaExtractor, 1);  // Fallback
```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM
**UrlConvertHelper**: URL normalization

---

## Development Workflow

	}

	disks := strings.Split(r.Form.Get("disks"), ",")
	byDisk := strings.EqualFold(r.Form.Get("by-disk"), "true")
	var diskMap map[string]struct{}
	if len(disks) > 0 && disks[0] != "" {
		diskMap = make(map[string]struct{}, len(disks))
		for _, k := range disks {
			if k != "" {
				diskMap[k] = struct{}{}
			}
		}
	}
	jobID := r.Form.Get("by-jobID")

	hosts := strings.Split(r.Form.Get("hosts"), ",")

                    rule.setResponseProcessor(container.getComponent("sitemapsResponseProcessor"));
                    rule.setRuleId("sitemapsRule");
                    rule.addRule("url", ".*sitemap.*");
                })//
                .<DefaultResponseProcessor> singleton("defaultResponseProcessor", DefaultResponseProcessor.class, processor -> {

        }
    }

    /**
     * Processes robots.txt for the given URL.
     * This method fetches and parses the robots.txt file to extract disallow/allow rules
     * and sitemap information.
     *
     * @param url The URL to process robots.txt for
     */
    protected void processRobotsTxt(final String url) {
        if (StringUtil.isBlank(url)) {

   *
   * @since 20.0
   */
  public static final MediaType VTT_UTF_8 = createConstantUtf8(TEXT_TYPE, "vtt");

  /* image types */
  /**
   * <a href="https://en.wikipedia.org/wiki/BMP_file_format">Bitmap file format</a> ({@code bmp}
   * files).
   *
   * @since 13.0
   */
  public static final MediaType BMP = createConstant(IMAGE_TYPE, "bmp");

  /**

				"image/x-freehand",
				"image/x-jp2-codestream",
				"image/x-jp2-container",
				"image/x-niff",
				"image/x-pcx",
				"image/x-pict",
				"image/x-portable-anymap",
				"image/x-portable-bitmap",
				"image/x-portable-graymap",
				"image/x-portable-pixmap",
				"image/x-raw-adobe",
				"image/x-raw-hasselblad",
				"image/x-raw-fuji",
				"image/x-raw-canon",
				"image/x-raw-kodak",

	// function type for creating signed requests - used to repeat
	// requests with V2 and V4 signing.
	type testSignedReqFn func(method, urlStr string, contentLength int64,
		body io.ReadSeeker, accessKey, secretKey string, metamap map[string]string) (*http.Request,
		error)

	mkGetReq := func(oi ObjectInput, byteRange string, i int, mkSignedReq testSignedReqFn) {
		object := oi.objectName
		rec := httptest.NewRecorder()

fsx950223, ganler, Gaurav Singh, Georgios Pinitas, Gian Marco Iodice, Giorgio
Arena, Giuseppe Rossini, Gregory Keith, Guozhong Zhuang, gurushantj, Hahn
Anselm, Harald Husum, Harjyot Bagga, Hristo Vrigazov, Ilya Persky, Ir1d, Itamar
Turner-Trauring, jacco, Jake Tae, Janosh Riebesell, Jason Zaman, jayanth, Jeff
Daily, Jens Elofsson, Jinzhe Zeng, JLZ, Jonas Skog, Jonathan Dekhtiar, Josh