*/
    @Override
    public void addIncludeUrlFilter(final String sessionId, final List<String> urlList) {
        final Set<String> invalidateSet = new HashSet<>();
        final List<OpenSearchUrlFilter> urlFilterList = new ArrayList<>(urlList.size());
        for (final String url : urlList) {
            final OpenSearchUrlFilter esUrlFilter = new OpenSearchUrlFilter();

     */
    protected void load(final CrawlerClientFactory crawlerClientFactory, final String regex, final String componentName) {
        if (logger.isDebugEnabled()) {
            logger.debug("loading {}", componentName);
        }
        try (CrawlerClient client = crawlerContainer.getComponent(componentName)) {
            crawlerClientFactory.addClient(regex, client);
        } catch (final Exception e) {

            return builder.execute();
        });
        final OpenSearchResultList<OpenSearchAccessResult> targetList = new OpenSearchResultList<>();
        final SearchHits hits = response.getHits();
        final TotalHits totalHits = hits.getTotalHits();
        final long totalHitsValue = totalHits != null ? totalHits.value() : 0;
        targetList.setTotalHits(totalHitsValue);

     */
    @Override
    public void process(final ResponseData responseData) {
        final SitemapsHelper sitemapsHelper = crawlerContainer.getComponent("sitemapsHelper");
        try (final InputStream responseBody = responseData.getResponseBody()) {
            final SitemapSet sitemapSet = sitemapsHelper.parse(responseBody);
            final Set<RequestData> requestDataSet = new LinkedHashSet<>();

        this.filterShards = filterShards;
    }

    /**
     * Returns the number of queue replicas.
     * @return The number of queue replicas.
     */
    public int getQueueReplicas() {
        return queueReplicas;
    }

    /**
     * Sets the number of queue replicas.
     * @param queueReplicas The number of queue replicas.
     */

    private static final long serialVersionUID = 1L;

    /**
     * Creates a new MimeTypeException with the specified detail message and cause.
     *
     * @param message the detail message explaining the reason for the exception
     * @param cause the underlying cause of the exception
     */
    public MimeTypeException(final String message, final Throwable cause) {
        super(message, cause);
    }

            ftpClient.doGet("ftp://localhost:" + FTP_PORT + "/");
            fail();
        } catch (final ChildUrlsException e) {
            final Set<RequestData> urlSet = e.getChildUrlList();
            assertEquals(5, urlSet.size());
            final List<String> urlList = urlSet.stream().map(x -> x.getUrl()).sorted().toList();
            assertEquals("ftp://localhost:10021/dir1", urlList.get(0));

     */
    protected void storeChildUrls(final Set<RequestData> childUrlList, final String url, final int depth) {
        if (crawlerContext.getMaxDepth() >= 0 && depth > crawlerContext.getMaxDepth()) {
            return;
        }

        // add url and filter
        final Set<String> urlSet = new HashSet<>();
        final List<UrlQueue<?>> childList = childUrlList.stream()

            return tempDir;
        } catch (final Exception e) {
            throw new CrawlerSystemException(e);
        }
    }

    private static void generateContents(final File dir, final int count) throws Exception {
        if (count <= 0) {
            return;
        }

        final String content = getHtmlContent(count);

        final File indexFile = new File(dir, "index.html");

     * @param file The FTP file object, or null if not found
     */
    protected void updateResponseData(final String uri, final boolean includeContent, final ResponseData responseData,
            final FTPClient client, final FtpInfo ftpInfo, final FTPFile file) {
        if (file == null) {
            responseData.setHttpStatusCode(Constants.NOT_FOUND_STATUS_CODE);