*   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.

 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

    /** Pattern for parsing user-agent records. */
    protected static final Pattern USER_AGENT_RECORD =
            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */

     */
    public void setCookieSpec(final String cookieSpec) {
        this.cookieSpec = cookieSpec;
    }

    /**
     * Sets the user agent string.
     *
     * @param userAgent The user agent string
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the proxy host name.
     *

```bash
# Format code (Eclipse formatter)
mvn formatter:format

# Check/apply license headers
mvn license:check
mvn license:format

# Generate test coverage report
mvn jacoco:prepare-agent test jacoco:report

# Generate API documentation
mvn javadoc:javadoc
```

### Testing

The project uses JUnit 4 with embedded OpenSearch for integration testing:

```bash
# Run all tests