/**
     * The feature for external general entities in XML.
     */
    public static final String FEATURE_EXTERNAL_GENERAL_ENTITIES = "http://xml.org/sax/features/external-general-entities";

    /**
     * Feature for external parameter entities in XML.
     */

    /** Indicates processing a child URL due to an exception. */
    PROCESS_CHILD_URL_BY_EXCEPTION,
    /** Indicates an access exception during crawling. */
    CRAWLING_ACCESS_EXCEPTION,
    /** Indicates a general exception during crawling. */
    CRAWLING_EXCEPTION,
    /** Indicates no URL is available in the queue. */
    NO_URL_IN_QUEUE,
    /** Indicates the start of a crawler thread. */
    START_THREAD,

     * both sources differently.
     */
    private String lastmod;

    /**
     * How frequently the page is likely to change. This value provides general
     * information to search engines and may not correlate exactly to how often
     * they crawl the page. Valid values are:
     * <ul>
     * <li>always</li>
     * <li>hourly</li>
     * <li>daily</li>

    }

    /**
     * Returns the most specific directive matching the given user agent.
     * The method finds the longest matching user agent pattern in the directives,
     * excluding the general "*" pattern which matches all bots.
     *
     * @param userAgent the user agent string to match against directives,
     *                 can be null (treated as empty string)