urlFilter.addInclude("https?://[^/]+\\.example\\.com/.*");

        assertTrue(urlFilter.match("http://www.example.com/"));
        assertTrue(urlFilter.match("https://api.example.com/v1/users"));
        assertTrue(urlFilter.match("http://subdomain.example.com/page"));
        assertFalse(urlFilter.match("https://example.org/"));
        assertFalse(urlFilter.match("ftp://files.example.com/"));
    }

    /**

#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/public/session.h"

namespace tensorflow {
class Device;
class DeviceMgr;
class ServerInterface;
}  // namespace tensorflow

// Internal structures used by the C API. These are likely to change and should
// not be depended on.

struct TF_SessionOptions {
  tensorflow::SessionOptions options;
};

struct TF_DeprecatedSession {
  tensorflow::Session* session;
};

- [ ] Provide integration tests (under `<subproject>/src/integTest`) to verify changes from a user perspective.
- [ ] Provide unit tests (under `<subproject>/src/test`) to verify logic.
- [ ] Update User Guide, DSL Reference, and Javadoc for public-facing changes.
- [ ] Ensure that tests pass sanity check: `./gradlew sanityCheck`.

 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *
 * <p>The class uses case-insensitive pattern matching for user agents and supports
 * wildcard characters (*) in user agent strings. When multiple directives match a user agent,
 * the most specific (longest) match is used.</p>
 *
 */
public class RobotsTxt {

        if (accessResult == null) {
            throw new CrawlerSystemException("AccessResult is null.");
        }

        synchronized (idCountLock) {
            idCount++;
            accessResult.setId(idCount);
            AccessResultData<Long> accessResultData = accessResult.getAccessResultData();
            if (accessResultData == null) {
                accessResultData = new AccessResultDataImpl<>();

    /**
     * The kind of suggest item.
     */
    public enum Kind {
        /** Document kind. */
        DOCUMENT("document"),
        /** Query kind. */
        QUERY("query"),
        /** User kind. */
        USER("user");

        private final String kind;

        /**
         * Constructor for Kind.
         * @param kind The kind string.
         */
        Kind(final String kind) {

     *
     * @return the unique identifier of type IDTYPE.
     */
    IDTYPE getId();

    /**
     * Sets the ID of the access result data.
     *
     * @param id the ID to set
     */
    void setId(IDTYPE id);

    /**
     * Retrieves the name of the transformer associated with this access result data.
     *
     * @return the name of the transformer.
     */
    String getTransformerName();

    /**

     */
    void setHttpStatusCode(Integer httpStatusCode);

    /**
     * Returns the HTTP method used for the access.
     *
     * @return the HTTP method
     */
    String getMethod();

    /**
     * Sets the HTTP method used for the access.
     *
     * @param method the HTTP method
     */
    void setMethod(String method);

    /**

    }

    @Override
    public IDTYPE getId() {
        return id;
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.entity.UrlQueue#setId(IDTYPE)
     */
    @Override
    public void setId(final IDTYPE id) {
        this.id = id;
    }

    /*
     * (non-Javadoc)
     *
     * @see org.codelibs.fess.crawler.entity.UrlQueue#getSessionId()
     */
    @Override

    public void setAuthSchemeProviderMap(final Map<String, AuthSchemeProvider> authSchemeProviderMap) {
        this.authSchemeProviderMap = authSchemeProviderMap;
    }

    /**
     * Sets the user agent string.
     * @param userAgent The user agent string.
     */
    public void setUserAgent(final String userAgent) {
        this.userAgent = userAgent;
    }

    /**
     * Sets the credentials provider.