<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />
	<include path="crawler/transformer.xml" />

	<component name="ruleManager" class="org.codelibs.fess.crawler.rule.impl.RuleManagerImpl" instance="prototype">
		<postConstruct name="addRule">
			<arg>sitemapsRule</arg>
		</postConstruct>
		<postConstruct name="addRule">

{"index":{"_index":"fess_config.scheduled_job","_id":"default_crawler"}}
{"name":"Default Crawler","target":"all","cronExpression":"0 0 * * *","scriptType":"groovy","scriptData":"return container.getComponent(\"crawlJob\").logLevel(\"info\").gcLogging().execute(executor);","jobLogging":true,"crawler":true,"available":true,"sortOrder":1,"createdBy":"system","createdTime":0,"updatedBy":"system","updatedTime":0}
{"index":{"_index":"fess_config.scheduled_job","_id":"suggest_indexer"}}

import org.codelibs.fess.crawler.entity.AccessResultImpl;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.entity.UrlQueueImpl;
import org.codelibs.fess.crawler.helper.MemoryDataHelper;
import org.codelibs.fess.crawler.service.UrlQueueService;

import jakarta.annotation.Resource;

/**
 * Implementation of the {@link UrlQueueService} interface.

		return errServerNotInitialized
	}

	started := tracker.Started
	if started.IsZero() || started.Equal(timeSentinel) {
		healingLogIf(ctx, fmt.Errorf("unexpected tracker healing start time found: %v", started))
		started = time.Time{}
	}

	// Final tracer update before quitting
	defer func() {
		tracker.setObject("")
		tracker.setBucket("")
		healingLogIf(ctx, tracker.update(ctx))
	}()

<components namespace="fessCrawler">
    <include path="crawler/container.xml"/>
    <include path="crawler/client.xml"/>
    <include path="crawler/rule.xml"/>
    <include path="crawler/filter.xml"/>
    <include path="crawler/interval.xml"/>
    <include path="crawler/extractor.xml"/>
    <include path="crawler/mimetype.xml"/>
    <include path="crawler/encoding.xml"/>
    <include path="crawler/urlconverter.xml"/>
    <include path="crawler/log.xml"/>

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.client;

import java.util.Map;

import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;

/**
 * Interface representing a client for a web crawler.
 * This client is responsible for executing requests and handling responses.

### Key File Locations

**Core**: `fess-crawler/src/main/java/org/codelibs/fess/crawler/`
- `Crawler.java`, `CrawlerContext.java`, `CrawlerThread.java`

**Clients**: `fess-crawler/src/main/java/org/codelibs/fess/crawler/client/`
- `http/HcHttpClient.java`, `fs/FileSystemClient.java`, `storage/StorageClient.java`

**DI Config**: `fess-crawler-lasta/src/main/resources/crawler.xml`

### Exception Hierarchy

```

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.builder.RequestDataBuilder;
import org.codelibs.fess.crawler.client.CrawlerClient;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.exception.ChildUrlsException;

import org.codelibs.core.lang.SystemUtil;
import org.codelibs.fess.crawler.Constants;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.container.CrawlerContainer;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.RequestData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.Crawler;
import org.codelibs.fess.crawler.CrawlerContext;
import org.codelibs.fess.crawler.CrawlerStatus;
import org.codelibs.fess.crawler.interval.FessIntervalController;
import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlFilterService;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;