* It uses Log4j2 for logging.
 *
 * <p>
 * The class contains methods for logging different types of events, such as:
 * </p>
 * <ul>
 *   <li>Starting and finishing threads</li>
 *   <li>Starting and cleaning up crawling</li>
 *   <li>Handling unsupported URLs</li>
 *   <li>Checking last modified dates</li>
 *   <li>Getting content</li>
 *   <li>Handling redirects</li>

        // Disallow: /fish$ - should block exactly /fish but not /fishing
        assertFalse(robotsTxt.allows("/fish", "EndPathBot"));
        assertTrue(robotsTxt.allows("/fishing", "EndPathBot"));
        assertTrue(robotsTxt.allows("/fish/", "EndPathBot"));

        // Disallow: /temp$ but Allow: /fishing
        assertFalse(robotsTxt.allows("/temp", "EndPathBot"));

            // If pattern doesn't end with $, it implicitly matches anything after it
            if (pattern.isEmpty() || pattern.charAt(pattern.length() - 1) != '$') {
                // This makes "/fish" match "/fish", "/fishing", "/fish/", etc.
                // No need to add anything - the pattern naturally continues
            }

            return Pattern.compile(regex.toString());
        }

        /**