būt  
biju 
biji
bija
bijām
bijāt
esmu
esi
esam
esat 
būšu     
būsi
būs
būsim
būsiet
tikt
tiku
tiki
tika
tikām
tikāt
tieku
tiec
tiek
tiekam
tiekat
tikšu
tiks
tiksim
tiksiet
tapt
tapi
tapāt
topat
tapšu
tapsi
taps
tapsim
tapsiet
kļūt
kļuvu
kļuvi
kļuva
kļuvām
kļuvāt

"taču", "nu", "pat", "tiklab", "iekšpus", "nedz", "tik", "nevis", "turpretim", "jeb", "iekam", "iekām", "iekāms", "kolīdz", "līdzko", "tiklīdz", "jebšu", "tālab", "tāpēc", "nekā", "itin", "jā", "jau", "jel", "nē", "nezin", "tad", "tikai", "vis", "tak", "iekams", "vien", "# modal verbs", "būt  ", "biju ", "biji", "bija", "bijām", "bijāt", "esmu", "esi", "esam", "esat ", "būšu     ", "būsi", "būs", "būsim", "būsiet", "tikt", "tiku", "tiki", "tika", "tikām", "tikāt", "tieku", "tiec", "tiek", "tiekam",...

"taču", "nu", "pat", "tiklab", "iekšpus", "nedz", "tik", "nevis", "turpretim", "jeb", "iekam", "iekām", "iekāms", "kolīdz", "līdzko", "tiklīdz", "jebšu", "tālab", "tāpēc", "nekā", "itin", "jā", "jau", "jel", "nē", "nezin", "tad", "tikai", "vis", "tak", "iekams", "vien", "# modal verbs", "būt  ", "biju ", "biji", "bija", "bijām", "bijāt", "esmu", "esi", "esam", "esat ", "būšu     ", "būsi", "būs", "būsim", "būsiet", "tikt", "tiku", "tiki", "tika", "tikām", "tikāt", "tieku", "tiec", "tiek", "tiekam",...

<?xml version="1.0" encoding="UTF-8"?>
<properties>
  <parsers>
    <parser class="org.apache.tika.parser.DefaultParser">
      <parser-exclude class="org.apache.tika.parser.ocr.TesseractOCRParser"/>
    </parser>
  </parsers>

			<artifactId>tika-parser-html-module</artifactId>
			<version>${tika.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.tika</groupId>
			<artifactId>tika-parser-image-module</artifactId>
			<version>${tika.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.tika</groupId>
			<artifactId>tika-parser-mail-module</artifactId>

			<groupId>com.ibm.icu</groupId>
			<artifactId>icu4j</artifactId>
			<version>${icu4j.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.tika</groupId>
			<artifactId>tika-langdetect-optimaize</artifactId>
			<version>${tika.version}</version>
			<exclusions>
				<exclusion>
					<groupId>javax.annotation</groupId>
					<artifactId>javax.annotation-api</artifactId>
				</exclusion>

		<property name="valueSeparator">"\\n"</property>
		<property name="roleSeparator">","</property>
		 -->
	</component>
	<component name="tikaConfig" class="org.apache.tika.config.TikaConfig">
		<arg>org.codelibs.fess.util.ResourceUtil.getConfOrClassesPath("tika.xml")</arg>
	</component>

import java.util.Arrays;
import java.util.Map;
import java.util.stream.Collectors;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.tika.language.detect.LanguageDetector;
import org.apache.tika.language.detect.LanguageResult;
import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.mylasta.direction.FessConfig;
import org.codelibs.fess.util.ComponentUtil;

				"application/x-js-taro",
				"application/x-tika-msoffice",
				"application/x-tika-msoffice-embedded",
				"application/x-tika-msoffice-embedded;format=ole10_native",
				"application/x-tika-msoffice-embedded;format=comp_obj",
				"application/x-tika-msworks-spreadsheet",
				"application/x-tika-ooxml",
				"application/x-tika-ooxml-protected",
				"application/x-tika-staroffice",
				"application/x-uc2-compressed",

- **License**: Apache 2.0
- **DI**: LastaFlute DI
- **Repo**: https://github.com/codelibs/fess-crawler

### Tech Stack

- **HTTP**: Apache HttpComponents 4.5+ and 5.x (switchable)
- **Extraction**: Apache Tika, POI, PDFBox
- **Testing**: JUnit 4, UTFlute, Mockito, Testcontainers
- **Storage**: In-memory (default), OpenSearch (optional)
- **Cloud**: AWS SDK v2 (S3), Google Cloud Storage

### Protocols