import org.dbflute.utflute.core.PlainTestCase;

public class CrawlerTest extends PlainTestCase {

    public Crawler crawler;

    public DataService dataService;

    public UrlQueueService urlQueueService;

    public FileTransformer fileTransformer;

    private StandardCrawlerContainer container;

    @Override
    protected void setUp() throws Exception {
        super.setUp();

 * It uses various services and components, such as {@link UrlQueueService}, {@link DataService},
 * {@link CrawlerContainer}, {@link LogHelper}, {@link CrawlerClientFactory}, and {@link CrawlerContext},
 * to perform its tasks.
 * </p>
 *
 * <p>
 * The crawling process involves the following steps:
 * </p>
 * <ol>
 *   <li>Fetching a URL from the queue using {@link UrlQueueService#poll(String)}.</li>

public class CrawlerTest extends LastaDiTestCase {

    @Resource
    private Crawler crawler;

    @Resource
    private DataService dataService;

    @Resource
    private UrlQueueService urlQueueService;

    @Resource
    private FileTransformer fileTransformer;

    @Resource
    private CrawlerContainer crawlerContainer;

    @Override
    protected String prepareConfigFile() {
        return "crawler.xml";

    private static final Logger logger = LogManager.getLogger(Crawler.class);

    /**
     * Service for managing URL queues during crawling.
     */
    @Resource
    protected UrlQueueService<UrlQueue<?>> urlQueueService;

    /**
     * Service for managing access result data.
     */
    @Resource
    protected DataService<AccessResult<?>> dataService;

    /**

        final CrawlerContext crawlerContext = CrawlingParameterUtil.getCrawlerContext();
        final UrlQueueService<UrlQueue<?>> urlQueueService = CrawlingParameterUtil.getUrlQueueService();
        if (!urlQueueService.visited(urlQueue)) {
            if (checkAccessCount(crawlerContext)) {
                final AccessResult<?> accessResult = createAccessResult(responseData, resultData);

    private LanguageHelper languageHelper;
    private FessConfig fessConfig;
    private SearchEngineClient searchEngineClient;
    private DataService<OpenSearchAccessResult> dataService;
    private UrlQueueService<OpenSearchUrlQueue> urlQueueService;
    private UrlFilterService urlFilterService;
    private ClickLogBhv clickLogBhv;
    private FavoriteLogBhv favoriteLogBhv;
    private IngestFactory ingestFactory;

    @Override

    @Resource
    protected DataService<OpenSearchAccessResult> dataService;

    /** Service for managing URL crawling queue */
    @Resource
    protected UrlQueueService<OpenSearchUrlQueue> urlQueueService;

    /** Service for URL filtering operations */
    @Resource
    protected UrlFilterService urlFilterService;

    /** Behavior class for click log operations */
    @Resource

import jakarta.annotation.PreDestroy;
import jakarta.annotation.Resource;

/**
 * OpenSearchUrlQueueService is an implementation of {@link UrlQueueService} for OpenSearch.
 *
 * @author shinsuke
 *
 */
public class OpenSearchUrlQueueService extends AbstractCrawlerService implements UrlQueueService<OpenSearchUrlQueue> {
    /**
     * Logger instance.
     */

        } catch (final Exception e) {
            logger.warn("Failed to delete UrlFilter for {}", sid, e);
        }

        try {
            // clear queue
            urlQueueService.clearCache();
            urlQueueService.delete(sid);
        } catch (final Exception e) {
            logger.warn("Failed to delete UrlQueue for {}", sid, e);
        }

        try {
            // clear