* It uses Log4j2 for logging.
 *
 * <p>
 * The class contains methods for logging different types of events, such as:
 * </p>
 * <ul>
 *   <li>Starting and finishing threads</li>
 *   <li>Starting and cleaning up crawling</li>
 *   <li>Handling unsupported URLs</li>
 *   <li>Checking last modified dates</li>
 *   <li>Getting content</li>
 *   <li>Handling redirects</li>

        // Disallow: /fish$ - should block exactly /fish but not /fishing
        assertFalse(robotsTxt.allows("/fish", "EndPathBot"));
        assertTrue(robotsTxt.allows("/fishing", "EndPathBot"));
        assertTrue(robotsTxt.allows("/fish/", "EndPathBot"));

        // Disallow: /temp$ but Allow: /fishing
        assertFalse(robotsTxt.allows("/temp", "EndPathBot"));

User-agent: WildcardBot
Disallow: /*.pdf$
Disallow: /admin/*.php
Disallow: /*/private/
Allow: /public/*.html

# Test end-of-path ($) patterns
User-agent: EndPathBot
Disallow: /fish$
Disallow: /temp$
Allow: /fishing

# Test complex patterns
User-agent: ComplexBot
Disallow: /
Allow: /$
Allow: /index.html$
Allow: /public/

# Test priority rules (longer match wins)
User-agent: PriorityBot
Disallow: /store

            // If pattern doesn't end with $, it implicitly matches anything after it
            if (pattern.isEmpty() || pattern.charAt(pattern.length() - 1) != '$') {
                // This makes "/fish" match "/fish", "/fishing", "/fish/", etc.
                // No need to add anything - the pattern naturally continues
            }

            return Pattern.compile(regex.toString());
        }

        /**