bf := bufio.NewReader(r)
	switch f := detect(bf); f {
	case formatGzip:
		gz, err := gzip.NewReader(bf)
		if err != nil {
			return err
		}
		defer gz.Close()
		r = gz
	case formatS2:
		r = s2.NewReader(bf)
	case formatZstd:
		// Limit to 16 MiB per stream.
		dec, err := zstd.NewReader(bf, zstd.WithDecoderMaxWindow(16<<20))

*.tar.bz2
parts/
prime/
stage/
.sia_temp/
config.json
node_modules/
mc.*
s3-check-md5*
xl-meta*
healing-*
inspect*.zip
200M*
hash-set
minio.RELEASE*
mc
nancy
inspects/*
.bin/
*.gz
docs/debugging/s3-verify/s3-verify
docs/debugging/xl-meta/xl-meta
docs/debugging/s3-check-md5/s3-check-md5
docs/debugging/hash-set/hash-set
docs/debugging/healing-bin/healing-bin
docs/debugging/inspect/inspect

			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>
		</postConstruct>
	</component>

	<component name="webHtmlRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"webHtmlRule"</property>

)

var gzipHandler = func() func(http.Handler) http.HandlerFunc {
	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
	if err != nil {
		// Static params, so this is very unlikely.
		logger.Fatal(err, "Unable to initialize server")
	}
	return gz
}()

// Set of handler options as bit flags
type hFlag uint8

const (

application/x-director         dcr              # Director File
application/x-dvi              dvi              # TeX dvi Format
application/x-gtar             gtar             # Gzip and Tar file
application/x-gzip             gz tgz           # Gzip and Tar file
application/x-compress         z                # Gzip and Tar file
application/x-hdf              hdf              # NCSA HDF
application/x-ica              ica              # WinFrames

Decompressing incompressible data has no significant performance impact.

Below is a list of common files and content-types which are typically not suitable for compression.

- Extensions

 | `gz`  | (GZIP)      |
 | `bz2` | (BZIP2)     |
 | `rar` | (WinRAR)    |
 | `zip` | (ZIP)       |
 | `7z`  | (7-Zip)     |
 | `xz`  | (LZMA)      |
 | `mp4` | (MP4)       |
 | `mkv` | (MKV media) |

        @DisplayName("Should return correct mime type for multiple extensions")
        @CsvSource({ "txt, text/plain", "css, text/css", "js, application/x-javascript", "zip, application/zip", "tar, application/x-tar",
                "gz, application/x-gzip", "tiff, image/tiff", "tif, image/tiff" })
        void testVariousExtensions(String extension, String expectedMimeType) throws IOException {
            assertEquals(expectedMimeType, mimeMap.getMimeType(extension));

### Protocols

HTTP/HTTPS, File, FTP/FTPS, SMB/CIFS (SMB1/SMB2+), Storage (MinIO via `storage://`), S3 (`s3://`), GCS (`gcs://`)

### Content Formats

Office (Word, Excel, PowerPoint), PDF, Archives (ZIP, TAR, GZ, LHA), HTML, XML, JSON, Markdown, Media metadata, Images (EXIF/IPTC/XMP), Email (EML)

---

## Architecture

### Module Structure

```
fess-crawler-parent/
├── fess-crawler/              # Core framework

	if !w.opts.Compress {
		return nil
	}

	oldLgFile := w.f.Name()
	r, err := os.Open(oldLgFile)
	if err != nil {
		return err
	}
	defer r.Close()

	gw, err := os.Create(oldLgFile + ".gz")
	if err != nil {
		return err
	}
	defer gw.Close()

	var wc io.WriteCloser = gzip.NewWriter(gw)
	if _, err = io.Copy(wc, r); err != nil {
		return err
	}

	if err = wc.Close(); err != nil {

    assertThat(Files.getFileExtension(".blah.txt")).isEqualTo("txt");
    assertThat(Files.getFileExtension("/tmp/blah.txt")).isEqualTo("txt");
    assertThat(Files.getFileExtension("blah.tar.gz")).isEqualTo("gz");
    assertThat(Files.getFileExtension("/")).isEqualTo("");
    assertThat(Files.getFileExtension(".")).isEqualTo("");
    assertThat(Files.getFileExtension("..")).isEqualTo("");