import org.codelibs.fess.crawler.util.OpenSearchCrawlerConfig;
import org.opensearch.action.DocWriteRequest.OpType;
import org.opensearch.index.query.QueryBuilders;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;

import jakarta.annotation.PostConstruct;

/**
 * OpenSearchUrlFilterService is an implementation of {@link UrlFilterService} for OpenSearch.
 */

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {