domainFilter.processUrl("http://example.com/");

        assertEquals(1, domainFilter.cachedIncludeSet.size());
        assertEquals(1, domainFilter.cachedExcludeSet.size());
        assertEquals("http://example.com/.*", domainFilter.cachedIncludeSet.toArray()[0]);
        assertEquals("http://example.com/.*", domainFilter.cachedExcludeSet.toArray()[0]);

        domainFilter.processUrl("https://test.com");

        assertEquals("http://www.example.com/", sitemaps[0].getLoc());
        assertEquals("monthly", ((SitemapUrl) sitemaps[0]).getChangefreq());
        assertEquals("0.8", ((SitemapUrl) sitemaps[0]).getPriority());

        assertNull(sitemaps[1].getLastmod());
        assertEquals("http://www.example.com/catalog?item=12&desc=vacation_hawaii", sitemaps[1].getLoc());

        rule.addCondition("url", "https?://.*\\.example\\.com/.*");

        ResponseData responseData1 = new ResponseData();
        responseData1.setUrl("http://www.example.com/page");
        assertTrue(rule.match(responseData1));

        ResponseData responseData2 = new ResponseData();
        responseData2.setUrl("https://api.example.com/v1/users");
        assertTrue(rule.match(responseData2));

株式会社ソニー・コンピュータエンタテインメント
〒108-8270 東京都港区港南1丁目6番27号

■"PlayStation Network"のご利用に関する規約
http://www.jp.playstation.com/R/psn_kiyaku

■特定商取引法に基づく表示
http://www.jp.playstation.com/R/aboutstore

■よくあるお問い合わせ（FAQ）
http://www.jp.playstation.com/support/

■株式会社ソニー・コンピュータエンタテインメントの個人情報保護方針
http://www.jp.playstation.com/about/privacy.html

        StatefulTransformer transformer = new StatefulTransformer("statefulTransformer");

        // Process multiple URLs
        String[] urls = { "http://example1.com", "http://example2.com", "http://example3.com" };

        for (String url : urls) {
            ResponseData responseData = new ResponseData();
            responseData.setUrl(url);

<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <sitemap>
      <loc>http://www.example.com/sitemap1.xml.gz</loc>
      <lastmod>2004-10-01T18:23:17+00:00</lastmod>
   </sitemap>
   <sitemap>
      <loc>http://www.example.com/sitemap2.xml.gz</loc>
      <lastmod>2005-01-01</lastmod>
   </sitemap>

        assertEquals("fileRule", rule.getRuleId());
    }

    public void test_getRule_sitemaps1() {
        final ResponseData responseData = new ResponseData();
        responseData.setUrl("http://www.example.com/sitemap1.xml");
        File file = ResourceUtil.getResourceAsFile("sitemaps/sitemap1.xml");
        responseData.setResponseBody(file, false);
        final Rule rule = ruleManager.getRule(responseData);

		</dependency>
		<dependency>
			<groupId>com.sun.mail</groupId>
			<artifactId>jakarta.mail</artifactId>
			<version>${jakarta.mail.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.pdfbox</groupId>
			<artifactId>jbig2-imageio</artifactId>
			<version>${jbig2.imageio.version}</version>
		</dependency>
		<dependency>
			<groupId>com.github.jai-imageio</groupId>

        urlQueue.setUrl("http://www.id1.com/");

        urlQueueService.insert(urlQueue);

        final OpenSearchUrlQueue urlQueue2 = new OpenSearchUrlQueue();
        urlQueue2.setCreateTime(System.currentTimeMillis());
        urlQueue2.setDepth(1);
        urlQueue2.setMethod("GET");
        urlQueue2.setSessionId("id2");
        urlQueue2.setUrl("http://www.id2.com/");

        urlQueueService.insert(urlQueue2);

import org.codelibs.fess.crawler.exception.RobotsTxtException;

/**
 * Robots.txt Specifications:
 * <ul>
 * <li><a href=
 * "https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt"
 * >https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
 * </a></li>
 * </ul>
 *
 * @author bowez
 * @author shinsuke
 *
 */
public class RobotsTxtHelper {