* either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler;

import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

/**
 * Constants used in the fess-crawler.
 * This class provides a collection of constant values for HTTP methods, status codes,

import java.util.concurrent.atomic.AtomicInteger;

import org.codelibs.fess.crawler.container.StandardCrawlerContainer;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.processor.ResponseProcessor;
import org.codelibs.fess.crawler.rule.Rule;
import org.codelibs.fess.crawler.rule.RuleManager;
import org.dbflute.utflute.core.PlainTestCase;

/**
 * Test class for AbstractRule.

 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
 * either express or implied. See the License for the specific language
 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.extractor.impl;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN" 
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<component name="crawlerConfig" class="org.codelibs.fess.crawler.util.FessCrawlerConfig">
		<property name="queueIndex">"fess_crawler.queue"</property>
		<property name="queueShards">5</property>
		<property name="queueReplicas">1</property>

     * Document URL patterns to exclude from search index.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String excludedDocUrls;

    /**
     * Additional configuration parameters for the crawler.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String configParameter;

    /**
     * The maximum crawling depth from the starting URLs.
     */
    @Min(value = 0)

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.builder;

import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.RequestData.Method;
import org.dbflute.utflute.core.PlainTestCase;

/**
 * Test class for {@link RequestDataBuilder}.
 */

* [LastaFlute](https://github.com/lastaflute/lastaflute "LastaFlute"): Framework Web
* [Lasta Job](https://github.com/lastaflute/lasta-job "Lasta Job"): Agendador de Tarefas
* [Fess Crawler](https://github.com/codelibs/fess-crawler "Fess Crawler"): Rastreador Web

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/transformer_basic.xml"/>

 */
package org.codelibs.fess.crawler.extractor.impl;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.codelibs.core.exception.IORuntimeException;
import org.codelibs.core.io.FileUtil;
import org.codelibs.fess.crawler.entity.ExtractData;

  private fun serverTruncatesRequestButTrailersCanStillBeRead(http2: Boolean) {
    val mockResponse =
      MockResponse
        .Builder()
        .doNotReadRequestBody()
        .trailers(headersOf("caboose", "xyz"))

    // Trailers always work for HTTP/2, but only for chunked bodies in HTTP/1.
    if (http2) {
      mockResponse.body("abc")
    } else {
      mockResponse.chunkedBody("abc", 1)
    }