*   <li>Processing responses</li>
 *   <li>Handling exceptions during crawling and child URL processing</li>
 *   <li>Handling cases where no URL is in the queue</li>
 *   <li>Handling cases where no response processor or rule is found</li>
 *   <li>Handling system errors</li>
 * </ul>
 *
 * <p>
 * Each logging method checks the log level before logging the message.
 * The log level can be configured in the Log4j2 configuration file.
 * </p>
 *

     * @return the extractor for the extension, or null if not found
     */
    private Extractor getExtractor(final String ext) {
        return extractorMap.get(ext);
    }

    /**
     * Gets the output extension for the specified input extension.
     *
     * @param extension the input file extension
     * @return the output extension, or "txt" if not found
     */

    /** Map of user agent patterns to their corresponding directives. */
    protected final Map<Pattern, Directive> directiveMap = new LinkedHashMap<>();

    /** List of sitemap URLs found in the robots.txt file. */
    private final List<String> sitemapList = new ArrayList<>();

    /**
     * Creates a new RobotsTxt instance.
     */
    public RobotsTxt() {
        // Default constructor
    }

        }
    }

    /**
     * Retrieves a setting value by key.
     * @param key The key of the setting.
     * @return The setting value, or null if not found.
     */
    public Object get(final String key) {
        final GetResponse getResponse =
                client.prepareGet().setIndex(settingsIndexName).setId(settingsId).execute().actionGet(getSearchTimeout());

        if (cachedXPathAPI == null) {
            cachedXPathAPI = new XPathAPI();
            xpathAPI.set(cachedXPathAPI);
        }
        return cachedXPathAPI;
    }

    /**
     * Stores child URLs found in the HTML content.
     *
     * @param responseData the response data containing the HTML content
     * @param resultData the result data to store child URLs in

            final String code = e.errorResponse().code();
            switch (code) {
            case "NoSuchBucket":
                throw new CrawlingAccessException("Bucket " + bucketName + " is not found.", e);
            case "NoSuchKey", "NoSuchObject":
                if (logger.isDebugEnabled()) {
                    logger.debug("{} is not an object.", path);
                }
                break;

    }

    /**
     * Gets the received date from a message by parsing the received headers.
     *
     * @param message the message to get the received date from
     * @return the received date or null if not found
     * @throws MessagingException if message access fails
     */
    protected static Date getReceivedDate(final Message message) throws MessagingException {
        final Date today = new Date();

 * </p>
 *
 * <p>
 * The class provides methods to check if a response is successful or not modified based on
 * configured HTTP status codes. It also handles the storage of child URLs found in the
 * response data, respecting the maximum depth and access count limits.
 * </p>
 *
 */
public class DefaultResponseProcessor implements ResponseProcessor {
    /** Logger instance for this class */

     * @param clazz The class of the object to retrieve.
     * @param sessionId The session ID of the document.
     * @param url The URL of the document.
     * @return The retrieved object, or null if not found.
     * @throws OpenSearchAccessException if the retrieval fails.
     */
    protected <T> T get(final Class<T> clazz, final String sessionId, final String url) {
        final String id = getId(sessionId, url);

            try {
                tikaConfig = crawlerContainer.getComponent("tikaConfig");
            } catch (final Exception e) {
                logger.debug("tikaConfig component is not found.", e);
            }
        }

        if (tikaConfig == null) {
            tikaConfig = TikaConfig.getDefaultConfig();
        }

        if (logger.isDebugEnabled()) {