Shinsuke Sugaya <******@****.***> 1444529815 +0900

    /** XPath expression for extracting robots content from meta tags */
    private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";

    /** Robots tag value indicating no indexing or following */
    private static final String ROBOTS_TAG_NONE = "none";

    /** Robots tag value indicating no indexing */

        dr.domain = n.getDomain();
        return dr;
    }

    /*
     * Split DFS path like \fs1.example.com\root5\link2\foo\bar.txt into at
     * most 3 components (not including the first index which is always empty):
     * result[0] = ""
     * result[1] = "fs1.example.com"
     * result[2] = "root5"
     * result[3] = "link2\foo\bar.txt"
     */

- **Testing**: JUnit 4, UTFlute, Mockito 5.7.0
- **Storage**: In-memory (default), OpenSearch (optional)

### Protocols

- **HTTP/HTTPS**: Full crawling, cookies, auth, robots.txt
- **File**: Local/network file systems
- **FTP**: With authentication
- **SMB/CIFS**: Windows shares (SMB1/SMB2+)
- **Storage**: MinIO/S3-compatible

### Content Formats

        robotsTxtHelper = container.getComponent("robotsTxtHelper");
    }

    public void testParse() {
        RobotsTxt robotsTxt;
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        try {
            robotsTxt = robotsTxtHelper.parse(in);
        } finally {
            CloseableUtil.closeQuietly(in);
        }

import java.util.Map;
import java.util.regex.Pattern;

import org.codelibs.core.lang.StringUtil;

/**
 * Represents a robots.txt file parser and handler.
 * This class manages the rules defined in a robots.txt file, including user agent directives,
 * allowed/disallowed paths, crawl delays, and sitemap URLs.
 *
 * <p>The robots.txt protocol is implemented according to the standard specification,

     * <p>
     * If a root package is specified at instance construction, only classes under the root package are targeted.
     * </p>
     *
     * @param handler the handler to process classes
     */
    void forEach(ClassHandler handler);

    /**
     * Searches for resources handled by this instance and calls the handler for each resource.
     * <p>

 * </ul>
 *
 * <p>References:</p>
 * <ul>
 * <li><a href="https://datatracker.ietf.org/doc/html/rfc9309">RFC 9309 - Robots Exclusion Protocol</a></li>
 * <li><a href="https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt">
 * Google's robots.txt Specification</a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

        // Add URLs to default set
        urlSet.add("http://example.com/robots.txt");
        urlSet.add("http://test.com/robots.txt");
        assertEquals(2, crawlerContext.getRobotsTxtUrlSet().size());

        // Set new set
        Set<String> newSet = new HashSet<>();
        newSet.add("http://new.com/robots.txt");
        crawlerContext.setRobotsTxtUrlSet(newSet);

    }

    /**
     * Constructs a new TransportException with the specified root cause.
     *
     * @param rootCause the root cause of this exception
     */
    public TransportException(final Throwable rootCause) {
        this.rootCause = rootCause;
    }

    /**
     * Constructs a new TransportException with the specified detail message and root cause.
     *
     * @param msg the detail message