* represents a valid sitemap. It uses a SitemapsHelper to validate the response body as an InputStream.
 * The rule checks if the URL matches the defined regex pattern and then validates the content as a sitemap.
 * If any exception occurs during the sitemap validation, it logs the error and returns false.
 *
 */
public class SitemapsRule extends RegexRule {
    /**
     * Serial version UID for serialization.

        final String content = ATTR_NAME + "," + testStr;
        final Map<String, String> params = new HashMap<String, String>();
        //final ExtractData text = extractor.getText(new ByteArrayInputStream(FileUtils.readFileToByteArray(new File(filePath))), params);
        final ExtractData text = extractor.getText(new ByteArrayInputStream(testStr.getBytes()), params);
        assertEquals(content, text.getContent());
    }

     */
    public static CrawlerContext getCrawlerContext() {
        return ROBOT_CONTEXT_THREAD_LOCAL.get();
    }

    /**
     * Sets the current {@link CrawlerContext} for the current thread.
     * If the provided {@code crawlerContext} is {@code null}, the context is removed from the thread-local storage.

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.exception;

/**
 * MaxLengthExceededException is thrown when the content length exceeds the maximum allowed length.
 * It extends CrawlingAccessException, indicating a crawling-related access issue.
 *
 */
public class MaxLengthExceededException extends CrawlingAccessException {

import java.io.IOException;
import java.util.Map;

import org.codelibs.core.misc.Base64Util;
import org.codelibs.fess.crawler.exception.OpenSearchAccessException;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

/**
 * OpenSearchAccessResultData is an implementation of {@link AccessResultData} for OpenSearch.
 */

 * governing permissions and limitations under the License.
 */
package org.codelibs.fess.crawler.entity;

import java.io.IOException;

import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;

/**
 * OpenSearchUrlQueue is an implementation of {@link UrlQueue} for OpenSearch.
 */
public class OpenSearchUrlQueue extends UrlQueueImpl<String> implements ToXContent {

    public static final String FS_FILE_GROUPS = "fsFileGroups";

    /** Character encoding for files */
    protected String charset = Constants.UTF_8;

    /** Helper for managing content length limits */
    @Resource
    protected ContentLengthHelper contentLengthHelper;

    /** Flag to track initialization status */
    protected AtomicBoolean isInit = new AtomicBoolean(false);

    /**

          documentation, if provided along with the Derivative Works; or,
          within a display generated by the Derivative Works, if and
          wherever such third-party notices normally appear. The contents
          of the NOTICE file are for informational purposes only and
          do not modify the License. You may add Your own attribution
          notices within Derivative Works that You distribute, alongside

    /** The CIFS context for NTLM authentication. */
    protected BaseContext cifsContext;

    /**
     * Constructs a JcifsEngine with the specified properties.
     *
     * @param props the properties for configuring the CIFS context
     * @throws CrawlingAccessException if an error occurs during initialization
     */

 * </p>
 * <ul>
 *   <li>Starting and finishing threads</li>
 *   <li>Starting and cleaning up crawling</li>
 *   <li>Handling unsupported URLs</li>
 *   <li>Checking last modified dates</li>
 *   <li>Getting content</li>
 *   <li>Handling redirects</li>
 *   <li>Processing responses</li>
 *   <li>Handling exceptions during crawling and child URL processing</li>
 *   <li>Handling cases where no URL is in the queue</li>