│   ├── entity/
│   │   ├── SuggestItem.java        # Core suggestion data structure
│   │   └── ElevateWord.java        # Promoted words configuration
│   ├── request/                    # Request/Response handling
│   │   ├── suggest/                # Suggestion requests
│   │   └── popularwords/           # Popular words requests
│   ├── index/                      # Indexing functionality

    protected String charsetName = Constants.UTF_8;

    /**
     * A directory to store downloaded files.
     */
    protected File baseDir;

    /**
     * Creates a file with the specified path, handling directory creation and duplicate names.
     *
     * @param path the file path to create
     * @return the created file
     * @throws CrawlerSystemException if directory creation fails
     */

 * Katakana string into a list of possible Alphabet readings. It uses a predefined mapping of Katakana
 * characters to their Alphabet equivalents, handling both single and double Katakana character combinations.
 * </p>
 *
 * <p>
 * The conversion process involves iterating through the input string, identifying Katakana characters,

 * and to parse an input stream into a {@link SitemapSet} object.
 * It uses SAX parser for XML sitemaps and XML sitemap indexes,
 * and handles potential exceptions during parsing.
 * The class also includes inner classes for handling XML sitemap and sitemap index parsing.
 */
public class SitemapsHelper {
    private static final Logger logger = LogManager.getLogger(SitemapsHelper.class);

- **Comprehensive Content Extraction**: Office documents, PDFs, archives, images, audio/video files
- **Multi-Threading**: Configurable thread pools for high-performance crawling
- **Fault Tolerance**: Built-in retry mechanisms and error handling
- **Flexible Configuration**: XML-based dependency injection with LastaFlute DI
- **Extensible Architecture**: Plugin system for custom extractors, transformers, and clients

 * and cache file size to optimize the extraction process.
 *
 * <p>
 * The main purpose of this class is to simplify the extraction process by providing a fluent interface
 * for configuring the extraction parameters and handling the underlying complexities of content processing,
 * such as MIME type detection, extractor selection, and content length validation.
 * </p>
 *
 * <p>
 * Example usage:
 * </p>
 *
 * <pre>
 * {@code

import org.codelibs.fess.crawler.exception.MaxLengthExceededException;

import jakarta.annotation.Resource;

/**
 * Abstract base class for CrawlerClient implementations.
 * Provides common functionality for handling initialization parameters,
 * content length checks, and default method implementations.
 * It defines the basic structure and configuration options for crawler clients.
 */

        assertFalse(urlFilter.match("https://example.com/document.pdf"));
        assertTrue(urlFilter.match("https://example.com/document.PDF"));
    }

    /**
     * Test very long URL handling
     */
    public void test_veryLongUrl() {
        String sessionId = "test-session-020";
        urlFilter.init(sessionId);

        // Create a very long URL

import jakarta.annotation.Resource;

/**
 * The Crawler class is the main class for web crawling. It manages the crawling process,
 * including adding URLs to the queue, filtering URLs, managing crawler threads,
 * and handling the overall crawling lifecycle.
 *
 * <p>It implements the Runnable interface to be executed in a separate thread,
 * and the AutoCloseable interface to ensure resources are properly released after use.
 *

 *
 * <p>Features:
 * <ul>
 *   <li>Automatic initialization of MinIO client</li>
 *   <li>Support for HEAD and GET operations</li>
 *   <li>Content length validation</li>
 *   <li>MIME type detection</li>
 *   <li>Handling of large files through temporary file storage</li>
 *   <li>Object metadata and tags retrieval</li>
 *   <li>Directory listing capabilities</li>
 * </ul>
 *