import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
import org.codelibs.fess.crawler.client.ftp.FtpClient;
import org.codelibs.fess.crawler.client.http.Authentication;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.http.form.FormScheme;
import org.codelibs.fess.crawler.client.http.impl.AuthenticationImpl;

import com.google.common.cache.LoadingCache;

import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;

/**
 * Helper class for managing crawler statistics and performance metrics.
 * This class provides functionality to track, record, and report statistics
 * about crawler operations including timing data, performance metrics, and
 * operational events. It uses an internal cache to maintain statistics

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.OpenSearchAccessResult;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.DataService;
import org.codelibs.fess.crawler.service.UrlFilterService;

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="intervalController"
		class="org.codelibs.fess.crawler.interval.FessIntervalController"
		instance="prototype">
	</component>

  fun readLine(): String {
    val line = source.readUtf8LineStrict(headerLimit)
    headerLimit -= line.length.toLong()
    return line
  }

  /** Reads headers or trailers. */
  fun readHeaders(): Headers {
    val result = Headers.Builder()
    while (true) {
      val line = readLine()
      if (line.isEmpty()) break
      result.addLenient(line)
    }
    return result.build()

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/transformer_basic.xml"/>

	<component name="fessXpathTransformer" class="org.codelibs.fess.crawler.transformer.FessXpathTransformer" instance="singleton">
		<property name="name">"fessXpathTransformer"</property>
		<property name="featureMap">defaultFeatureMap</property>

  private fun serverTruncatesRequestButTrailersCanStillBeRead(http2: Boolean) {
    val mockResponse =
      MockResponse
        .Builder()
        .doNotReadRequestBody()
        .trailers(headersOf("caboose", "xyz"))

    // Trailers always work for HTTP/2, but only for chunked bodies in HTTP/1.
    if (http2) {
      mockResponse.body("abc")
    } else {
      mockResponse.chunkedBody("abc", 1)
    }

index.document.suggest.index=fess
# Index name for crawler documents.
index.document.crawler.index=fess_crawler
# Number of primary shards for crawler queue index.
index.document.crawler.queue.number_of_shards=10
# Number of primary shards for crawler data index.
index.document.crawler.data.number_of_shards=10
# Number of primary shards for crawler filter index.
index.document.crawler.filter.number_of_shards=10

  for (pushPromise in pushPromises) {
    result.addPush(pushPromise.wrap())
  }

  result.settings(settings)
  result.status(status)
  result.headers(headers)
  result.trailers(trailers)

  when (socketPolicy) {
    SocketPolicy.EXPECT_CONTINUE, SocketPolicy.CONTINUE_ALWAYS -> result.add100Continue()
    SocketPolicy.UPGRADE_TO_SSL_AT_END -> result.inTunnel()

    }

    /** Wait time in milliseconds when crawler is not running */
    protected long crawlerWaitMillis = 10000;

    /** List of interval rules for controlling crawler timing */
    protected List<IntervalRule> ruleList = new ArrayList<>();

    /**
     * Checks the crawler status and waits if the crawler is not running.
     * This method blocks until the crawler is running again.
     */