*   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

        RobotsTxt robotsTxt;
        final InputStream in = new java.io.ByteArrayInputStream(robotsTxtContent.getBytes());
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

        // Directives before User-agent should be ignored

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

            logger.debug("replace: {} -> {}", url, newUrl);
        }
        return newUrl;
    }

    /**
     * Checks if the user agent matches the path mapping.
     *
     * @param pathMapping the path mapping
     * @return true if the user agent matches
     */
    protected boolean matchUserAgent(final PathMapping pathMapping) {
        if (!pathMapping.hasUAMathcer()) {
            return true;

    // ============================================================
    // User Agent Configuration
    // ============================================================

    /** Prefix for Fess crawler user agent string. */
    public static final String CRAWLING_USER_AGENT_PREFIX = "Mozilla/5.0 (compatible; Fess/";

    /** Suffix for Fess crawler user agent string with bot information URL. */

     */
    public void setCookieSpec(final String cookieSpec) {
        this.cookieSpec = cookieSpec;
    }

    /**
     * Sets the user agent string.
     *
     * @param userAgent The user agent string
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the proxy host name.
     *

			</plugin>
			<plugin>
				<groupId>org.jacoco</groupId>
				<artifactId>jacoco-maven-plugin</artifactId>
				<version>0.8.13</version>
				<executions>
					<execution>
						<goals>
							<goal>prepare-agent</goal>
						</goals>
					</execution>
					<execution>
						<id>report</id>
						<phase>prepare-package</phase>
						<goals>
							<goal>report</goal>
						</goals>
					</execution>

			</plugin>
			<plugin>
				<groupId>org.jacoco</groupId>
				<artifactId>jacoco-maven-plugin</artifactId>
				<version>0.8.13</version>
				<executions>
					<execution>
						<goals>
							<goal>prepare-agent</goal>
						</goals>
					</execution>
					<execution>
						<id>report</id>
						<phase>prepare-package</phase>
						<goals>
							<goal>report</goal>
						</goals>
					</execution>

mvn test -Dtest=SuggesterTest           # Run specific test
mvn package                              # Package JAR
mvn formatter:format license:format     # Format code and apply licenses
mvn clean jacoco:prepare-agent test jacoco:report  # Generate coverage report
```

### Adding New Features

1. Read related source files and tests
2. Write implementation following existing patterns
3. Add comprehensive tests

extractorFactory.addExtractor("text/html", htmlExtractor, 2);  // Weight 2
extractorFactory.addExtractor("text/html", tikaExtractor, 1);  // Fallback
```

### Helpers

**RobotsTxtHelper**: RFC 9309 parsing, user-agent matching, crawl-delay, sitemaps
**SitemapsHelper**: Sitemap XML parsing, index handling
**MimeTypeHelper**: MIME detection via Tika
**EncodingHelper**: Charset detection with BOM
**UrlConvertHelper**: URL normalization