*.tar.bz2
parts/
prime/
stage/
.sia_temp/
config.json
node_modules/
mc.*
s3-check-md5*
xl-meta*
healing-*
inspect*.zip
200M*
hash-set
minio.RELEASE*
mc
nancy
inspects/*
.bin/
*.gz
docs/debugging/s3-verify/s3-verify
docs/debugging/xl-meta/xl-meta
docs/debugging/s3-check-md5/s3-check-md5
docs/debugging/hash-set/hash-set
docs/debugging/healing-bin/healing-bin
docs/debugging/inspect/inspect

        // we always include all source files, and exclude what should not be checked
        .include("**")
        // exclude known binary extensions
        .exclude("**/*.gz")
        .exclude("**/*.ico")
        .exclude("**/*.jar")
        .exclude("**/*.zip")
        .exclude("**/*.jks")
        .exclude("**/*.crt")
        .exclude("**/*.keystore")
        .exclude("**/*.png")

			</component>
		</property>
		<postConstruct name="addRule">
			<arg>"url"</arg>
			<arg>"http[s]?:.*sitemap[^/]*\.xml.*|http[s]?:.*sitemap[^/]*\.gz.*|http[s]?:.*sitemap[^/]*\.txt.*"</arg>
		</postConstruct>
	</component>

	<component name="webHtmlRule" class="org.codelibs.fess.crawler.rule.impl.RegexRule" >
		<property name="ruleId">"webHtmlRule"</property>

application/x-director         dcr              # Director File
application/x-dvi              dvi              # TeX dvi Format
application/x-gtar             gtar             # Gzip and Tar file
application/x-gzip             gz tgz           # Gzip and Tar file
application/x-compress         z                # Gzip and Tar file
application/x-hdf              hdf              # NCSA HDF
application/x-ica              ica              # WinFrames

)

var gzipHandler = func() func(http.Handler) http.HandlerFunc {
	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
	if err != nil {
		// Static params, so this is very unlikely.
		logger.Fatal(err, "Unable to initialize server")
	}
	return gz
}()

// Set of handler options as bit flags
type hFlag uint8

const (

application/x-director         dcr              # Director File
application/x-dvi              dvi              # TeX dvi Format
application/x-gtar             gtar             # Gzip and Tar file
application/x-gzip             gz tgz           # Gzip and Tar file
application/x-compress         z                # Gzip and Tar file
application/x-hdf              hdf              # NCSA HDF
application/x-ica              ica              # WinFrames

        String artifactFileName = distFile.getName();
        String artifactName = artifactFileName.contains("oss") ? "elasticsearch-oss" : "elasticsearch";

        String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName.substring(artifactFileName.length() - 3);
        int x86ArchIndex = artifactFileName.indexOf("x86_64");
        int aarch64ArchIndex = artifactFileName.indexOf("aarch64");

Decompressing incompressible data has no significant performance impact.

Below is a list of common files and content-types which are typically not suitable for compression.

- Extensions

 | `gz`  | (GZIP)      |
 | `bz2` | (BZIP2)     |
 | `rar` | (WinRAR)    |
 | `zip` | (ZIP)       |
 | `7z`  | (7-Zip)     |
 | `xz`  | (LZMA)      |
 | `mp4` | (MP4)       |
 | `mkv` | (MKV media) |

        @DisplayName("Should return correct mime type for multiple extensions")
        @CsvSource({ "txt, text/plain", "css, text/css", "js, application/x-javascript", "zip, application/zip", "tar, application/x-tar",
                "gz, application/x-gzip", "tiff, image/tiff", "tif, image/tiff" })
        void testVariousExtensions(String extension, String expectedMimeType) throws IOException {
            assertEquals(expectedMimeType, mimeMap.getMimeType(extension));

### Protocols

HTTP/HTTPS, File, FTP/FTPS, SMB/CIFS (SMB1/SMB2+), Storage (MinIO via `storage://`), S3 (`s3://`), GCS (`gcs://`)

### Content Formats

Office (Word, Excel, PowerPoint), PDF, Archives (ZIP, TAR, GZ, LHA), HTML, XML, JSON, Markdown, Media metadata, Images (EXIF/IPTC/XMP), Email (EML)

---

## Architecture

### Module Structure

```
fess-crawler-parent/
├── fess-crawler/              # Core framework