</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>
		</postConstruct>
	</component>

	<component name="webHtmlRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"webHtmlRule"</property>

)

var gzipHandler = func() func(http.Handler) http.HandlerFunc {
	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
	if err != nil {
		// Static params, so this is very unlikely.
		logger.Fatal(err, "Unable to initialize server")
	}
	return gz
}()

// Set of handler options as bit flags
type hFlag uint8

const (

application/x-director         dcr              # Director File
application/x-dvi              dvi              # TeX dvi Format
application/x-gtar             gtar             # Gzip and Tar file
application/x-gzip             gz tgz           # Gzip and Tar file
application/x-compress         z                # Gzip and Tar file
application/x-hdf              hdf              # NCSA HDF
application/x-ica              ica              # WinFrames

Decompressing incompressible data has no significant performance impact.

Below is a list of common files and content-types which are typically not suitable for compression.

- Extensions

 | `gz`  | (GZIP)      |
 | `bz2` | (BZIP2)     |
 | `rar` | (WinRAR)    |
 | `zip` | (ZIP)       |
 | `7z`  | (7-Zip)     |
 | `xz`  | (LZMA)      |
 | `mp4` | (MP4)       |
 | `mkv` | (MKV media) |

        @DisplayName("Should return correct mime type for multiple extensions")
        @CsvSource({ "txt, text/plain", "css, text/css", "js, application/x-javascript", "zip, application/zip", "tar, application/x-tar",
                "gz, application/x-gzip", "tiff, image/tiff", "tif, image/tiff" })
        void testVariousExtensions(String extension, String expectedMimeType) throws IOException {
            assertEquals(expectedMimeType, mimeMap.getMimeType(extension));

### Protocols

HTTP/HTTPS, File, FTP/FTPS, SMB/CIFS (SMB1/SMB2+), Storage (MinIO via `storage://`), S3 (`s3://`), GCS (`gcs://`)

### Content Formats

Office (Word, Excel, PowerPoint), PDF, Archives (ZIP, TAR, GZ, LHA), HTML, XML, JSON, Markdown, Media metadata, Images (EXIF/IPTC/XMP), Email (EML)

---

## Architecture

### Module Structure

```
fess-crawler-parent/
├── fess-crawler/              # Core framework

    assertThat(Files.getFileExtension(".blah.txt")).isEqualTo("txt");
    assertThat(Files.getFileExtension("/tmp/blah.txt")).isEqualTo("txt");
    assertThat(Files.getFileExtension("blah.tar.gz")).isEqualTo("gz");
    assertThat(Files.getFileExtension("/")).isEqualTo("");
    assertThat(Files.getFileExtension(".")).isEqualTo("");
    assertThat(Files.getFileExtension("..")).isEqualTo("");

    assertThat(MoreFiles.getFileExtension(root().resolve("tmp/blah.txt"))).isEqualTo("txt");
    assertThat(MoreFiles.getFileExtension(FS.getPath("blah.tar.gz"))).isEqualTo("gz");
    assertThat(MoreFiles.getFileExtension(root())).isEqualTo("");
    assertThat(MoreFiles.getFileExtension(FS.getPath("."))).isEqualTo("");
    assertThat(MoreFiles.getFileExtension(FS.getPath(".."))).isEqualTo("");

    assertThat(MoreFiles.getFileExtension(root().resolve("tmp/blah.txt"))).isEqualTo("txt");
    assertThat(MoreFiles.getFileExtension(FS.getPath("blah.tar.gz"))).isEqualTo("gz");
    assertThat(MoreFiles.getFileExtension(root())).isEqualTo("");
    assertThat(MoreFiles.getFileExtension(FS.getPath("."))).isEqualTo("");
    assertThat(MoreFiles.getFileExtension(FS.getPath(".."))).isEqualTo("");

- RTF, WordPerfect

#### PDFs and Images
- PDF documents (text and metadata extraction)
- Images (JPEG, PNG, GIF, TIFF, BMP)
- Image metadata (EXIF, IPTC, XMP)

#### Archives and Compressed Files
- ZIP, TAR, GZ archives
- LHA compression format
- Nested archive extraction

#### Web and Markup
- HTML, XHTML with XPath support
- XML documents
- JSON and structured data

#### Media Files