if (StringUtil.isNotBlank(accessResultData.getEncoding())) {
            is.setEncoding(accessResultData.getEncoding());
        }

        // create handler
        final DocHandler handler = new DocHandler();

        // create a sax instance
        final SAXParserFactory spfactory = SAXParserFactory.newInstance();
        try {

import org.codelibs.fess.crawler.helper.SitemapsHelper;
import org.codelibs.fess.crawler.processor.ResponseProcessor;

import jakarta.annotation.Resource;

/**
 * A response processor implementation that handles sitemaps.
 * It parses the response body as a SitemapSet, extracts URLs from the sitemaps,
 * and adds them as child URLs to be crawled.
 *
 * <p>
 * This class uses a {@link SitemapsHelper} to parse the sitemap XML or text.

        Logger rootLogger = Logger.getLogger("");
        rootLogger.setLevel(Level.ALL);
        ConsoleHandler handler = new ConsoleHandler();
        handler.setFormatter(new SimpleFormatter());
        handler.setLevel(Level.ALL);
        rootLogger.addHandler(handler);
    }

    @Override
    public void tearDown() throws Exception {
        super.tearDown();
        // close runner

import jakarta.annotation.Resource;

/**
 * Abstract base class for implementing {@link Rule} interfaces.
 * Provides common functionality and properties for crawler rules.
 *
 * <p>
 * This class handles the registration of rules with the {@link RuleManager}
 * and provides getter and setter methods for common properties such as
 * {@code ruleId} and {@code responseProcessor}.
 * </p>
 *
 * <p>

import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *

/**
 * An abstract base class for implementing Extractor interfaces.
 * Provides common functionality such as access to CrawlerContainer components
 * and registration with the ExtractorFactory.
 *
 * <p>
 * This class handles the retrieval of essential crawler components like
 * {@link MimeTypeHelper} and {@link ExtractorFactory} from the
 * {@link CrawlerContainer}. It also provides a convenient method for

/**
 * The HankakuKanaToZenkakuKana class implements the Normalizer interface to convert
 * half-width Katakana characters (Hankaku Kana) to full-width Katakana characters (Zenkaku Kana).
 * It also handles the conversion of voiced and semi-voiced sound marks.
 *
 * <p>This class provides the following functionalities:</p>
 * <ul>
 *   <li>Normalization of a string containing Hankaku Kana to Zenkaku Kana.</li>

 * It processes the response data based on the HTTP status code and configured transformer.
 * </p>
 *
 * <p>
 * It handles successful responses by transforming the data using a {@link Transformer}
 * and storing the result. It also handles "Not Modified" responses by creating an empty
 * result and storing it. Unsuccessful responses are logged for debugging purposes.
 * </p>
 *
 * <p>

import org.codelibs.fess.crawler.container.CrawlerContainer;

import jakarta.annotation.Resource;

/**
 * Creates and manages crawler clients for web crawling operations.
 * This class handles the registration and loading of crawler client factories and their associated clients.
 * <p>
 * The class maintains a mapping between regular expressions and component names, and manages a list

 * from FTP servers. The client supports various configurations, including authentication, timeouts,
 * passive/active modes, and encoding settings. It also handles FTP file attributes such as user,
 * group, and symbolic links.
 *
 * <p>
 * The class uses Apache Commons Net library for FTP communication. It maintains a queue of FTPClient