|-----------|------------|
| Web Framework | LastaFlute (MVC framework) |
| DI Container | Lasta Di |
| Data Access | DBFlute (type-safe ORM for OpenSearch) |
| Search Engine | OpenSearch |
| App Server | Embedded Tomcat |
| Crawler | fess-crawler library |
| Scheduler | Lasta Job |
| Logging | Log4j2 |
| Testing | JUnit 4/5, UTFlute, REST Assured |

## Development Commands

### Setup
```bash

import org.codelibs.fess.Constants;
import org.codelibs.fess.crawler.entity.AccessResultData;
import org.codelibs.fess.crawler.entity.ExtractData;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.ResultData;
import org.codelibs.fess.crawler.entity.UrlQueue;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.util;

/**
 * Configuration class for OpenSearch crawler settings.
 * This class provides configuration for index names, shards, and replicas
 * for the queue, data, and filter indices used by the crawler.
 */
public class OpenSearchCrawlerConfig {
    /**
     * Constructs a new OpenSearchCrawlerConfig.
     */

import org.codelibs.fess.crawler.client.http.form.Hc4FormScheme;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.entity.RobotsTxt;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;
import org.codelibs.fess.crawler.exception.CrawlingAccessException;
import org.codelibs.fess.crawler.exception.MaxLengthExceededException;

import org.codelibs.fess.app.service.RequestHeaderService;
import org.codelibs.fess.app.service.WebAuthenticationService;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.http.HcHttpClient;
import org.codelibs.fess.crawler.client.http.config.WebAuthenticationConfig;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.mylasta.direction.FessConfig;

    public static final String CRAWLER_START_TIME = "CrawlerStartTime";

    /** Info map key for crawler end time. */
    public static final String CRAWLER_END_TIME = "CrawlerEndTime";

    /** Info map key for crawler execution time. */
    public static final String CRAWLER_EXEC_TIME = "CrawlerExecTime";

    /** Info map key for web/file system crawler start time. */

import org.codelibs.core.lang.StringUtil;
import org.codelibs.fess.Constants;
import org.codelibs.fess.app.service.FileAuthenticationService;
import org.codelibs.fess.crawler.client.CrawlerClientFactory;
import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
import org.codelibs.fess.helper.SystemHelper;
import org.codelibs.fess.opensearch.config.bsentity.BsFileConfig;

labels.crawling_info_delete_all_cancel=Cancel
labels.crawling_info_thread_dump=Thread Dump
labels.crawling_info_CrawlerStartTime=Crawler Start Time
labels.crawling_info_CrawlerEndTime=Crawler End Time
labels.crawling_info_CrawlerExecTime=Crawler Execution Time
labels.crawling_info_CrawlerStatus=Crawler Status
labels.crawling_info_WebFsCrawlExecTime=Crawl Execution Time (Web/File)

 */
package org.codelibs.fess.crawler.service.impl;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.codelibs.fess.crawler.entity.OpenSearchUrlFilter;
import org.codelibs.fess.crawler.exception.CrawlerSystemException;

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.transformer;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.codelibs.fess.crawler.entity.ResponseData;
import org.codelibs.fess.crawler.extractor.Extractor;
import org.codelibs.fess.crawler.extractor.ExtractorFactory;
import org.codelibs.fess.exception.FessSystemException;