domainFilter.processUrl("http://example.com/");

        assertEquals(1, domainFilter.cachedIncludeSet.size());
        assertEquals(1, domainFilter.cachedExcludeSet.size());
        assertEquals("http://example.com/.*", domainFilter.cachedIncludeSet.toArray()[0]);
        assertEquals("http://example.com/.*", domainFilter.cachedExcludeSet.toArray()[0]);

        domainFilter.processUrl("https://test.com");

        assertEquals("http://www.example.com/", sitemaps[0].getLoc());
        assertEquals("monthly", ((SitemapUrl) sitemaps[0]).getChangefreq());
        assertEquals("0.8", ((SitemapUrl) sitemaps[0]).getPriority());

        assertNull(sitemaps[1].getLastmod());
        assertEquals("http://www.example.com/catalog?item=12&desc=vacation_hawaii", sitemaps[1].getLoc());

        rule.addCondition("url", "https?://.*\\.example\\.com/.*");

        ResponseData responseData1 = new ResponseData();
        responseData1.setUrl("http://www.example.com/page");
        assertTrue(rule.match(responseData1));

        ResponseData responseData2 = new ResponseData();
        responseData2.setUrl("https://api.example.com/v1/users");
        assertTrue(rule.match(responseData2));

        StatefulTransformer transformer = new StatefulTransformer("statefulTransformer");

        // Process multiple URLs
        String[] urls = { "http://example1.com", "http://example2.com", "http://example3.com" };

        for (String url : urls) {
            ResponseData responseData = new ResponseData();
            responseData.setUrl(url);

        assertEquals("fileRule", rule.getRuleId());
    }

    public void test_getRule_sitemaps1() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/sitemap1.xml");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml");
        responseData.setResponseBody(file, false);
        final Rule rule = ruleManager.getRule(responseData);

        urlQueue.setUrl("http://www.id1.com/");

        urlQueueService.insert(urlQueue);

        final OpenSearchUrlQueue urlQueue2 = new OpenSearchUrlQueue();
        urlQueue2.setCreateTime(System.currentTimeMillis());
        urlQueue2.setDepth(1);
        urlQueue2.setMethod("GET");
        urlQueue2.setSessionId("id2");
        urlQueue2.setUrl("http://www.id2.com/");

        urlQueueService.insert(urlQueue2);

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {

        CrawlingParameterUtil.setUrlQueue(new UrlQueueImpl());
        final UrlQueue q = CrawlingParameterUtil.getUrlQueue();
        for (int i = 0; i < numTasks; i++) {
            q.setUrl("http://example.com");
        }

        final HostIntervalController controller = new HostIntervalController();
        controller.delayMillisBeforeProcessing = waittime;
        controller.delayMillisAfterProcessing = 0L;

        responseData.setUrl("http://example.com/test");

        ResultData result = transformer.transform(responseData);

        // Verify
        assertNotNull(result);
        assertEquals("customName", result.getTransformerName());
        assertEquals(1, transformer.getTransformCallCount());
        assertEquals("http://example.com/test", transformer.getLastResponseData().getUrl());

 * <pre>
 * {@code
 * IdnDnsResolver resolver = new IdnDnsResolver();
 * resolver.setFlag(IDN.ALLOW_UNASSIGNED);
 * resolver.setEncoding("UTF-8");
 * InetAddress[] addresses = resolver.resolve("example.com");
 * }
 * </pre>
 *
 */
public class IdnDnsResolver implements DnsResolver {

    /** Flag for IDN conversion. */
    protected int flag = 0;

    /** Encoding for URL decoding. */