* [LastaFlute](https://github.com/lastaflute/lastaflute "LastaFlute") : Framework Web
* [Lasta Job](https://github.com/lastaflute/lasta-job "Lasta Job") : Planificateur de tâches
* [Fess Crawler](https://github.com/codelibs/fess-crawler "Fess Crawler") : Crawler Web

/*
 [Crawler Notification]
 Crawler notification mail.
*/
subject: [FESS] Crawler completed: /*pmb.hostname*/
>>>
--- Server Info ---
Host Name: /*pmb.hostname:orElse('Unknown')*/
Job Name: /*pmb.jobname:orElse('Unknown')*/

--- Web/FileSystem Crawler ---
Start Time: /*pmb.webFsCrawlStartTime:orElse('-')*/
End Time:   /*pmb.webFsCrawlEndTime:orElse('-')*/
Exec Time:  /*pmb.webFsCrawlExecTime:orElse('-')*/ ms

--- Web/FileSystem Indexer ---

 */
package org.codelibs.fess.crawler.processor;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.AccessResult;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.processor.impl.DefaultResponseProcessor;

|-----------|------------|
| Web Framework | LastaFlute (MVC framework) |
| DI Container | Lasta Di |
| Data Access | DBFlute (type-safe ORM for OpenSearch) |
| Search Engine | OpenSearch |
| App Server | Embedded Tomcat |
| Crawler | fess-crawler library |
| Scheduler | Lasta Job |
| Logging | Log4j2 |
| Testing | JUnit 4/5, UTFlute, REST Assured |

## Development Commands

### Setup
```bash

* [LastaFlute](https://github.com/lastaflute/lastaflute "LastaFlute"): Web-Framework
* [Lasta Job](https://github.com/lastaflute/lasta-job "Lasta Job"): Job-Scheduler
* [Fess Crawler](https://github.com/codelibs/fess-crawler "Fess Crawler"): Web-Crawler

     * Document URL patterns to exclude from search index.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String excludedDocUrls;

    /**
     * Additional configuration parameters for the crawler.
     */
    @CustomSize(maxKey = "form.admin.max.input.size")
    public String configParameter;

    /**
     * The maximum crawling depth from the starting URLs.
     */
    @Min(value = 0)

{"index":{"_index":"fess_config.scheduled_job","_id":"default_crawler"}}
{"name":"Default Crawler","target":"all","cronExpression":"0 0 * * *","scriptType":"groovy","scriptData":"return container.getComponent(\"crawlJob\").logLevel(\"info\").gcLogging().execute(executor);","jobLogging":true,"crawler":true,"available":true,"sortOrder":1,"createdBy":"system","createdTime":0,"updatedBy":"system","updatedTime":0}
{"index":{"_index":"fess_config.scheduled_job","_id":"suggest_indexer"}}

<!DOCTYPE components PUBLIC "-//DBFLUTE//DTD LastaDi 1.0//EN"
	"http://dbflute.org/meta/lastadi10.dtd">
<components namespace="fessCrawler">
	<include path="crawler/container.xml" />
	<include path="crawler/transformer.xml" />

	<component name="ruleManager" class="org.codelibs.fess.crawler.rule.impl.RuleManagerImpl" instance="prototype">
		<postConstruct name="addRule">
			<arg>sitemapsRule</arg>
		</postConstruct>
		<postConstruct name="addRule">

    private static String crawlLabelId;

    @BeforeAll
    protected static void initAll() {
        RestAssured.baseURI = getFessUrl();
        settingTestToken();

        // Create and execute a file crawler
        crawlLabelId = createCrawlLabel();

        createFileConfig();
        logger.info("FileConfig is created");
        refresh();
        fileConfigId = getFileConfigIds(NAME_PREFIX).get(0);

 */
package org.codelibs.fess.crawler.service;

import java.util.List;
import java.util.Map;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.OpenSearchUrlQueue;
import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;
import org.codelibs.fess.crawler.util.OpenSearchCrawlerConfig;