fileTransformer = new FileTransformer();
        fileTransformer.setName("fileTransformer");
        Map<String, String> featureMap = newHashMap();
        featureMap.put("http://xml.org/sax/features/namespaces", "false");
        fileTransformer.setFeatureMap(featureMap);
        Map<String, String> propertyMap = newHashMap();
        fileTransformer.setPropertyMap(propertyMap);
        Map<String, String> childUrlRuleMap = newHashMap();

     */
    void saveSession(String sessionId);

    /**
     * Checks if a URL has been visited.
     *
     * @param urlQueue The URL queue.
     * @return true if the URL has been visited, otherwise false.
     */
    boolean visited(QUEUE urlQueue);

    /**
     * Generates URL queues.
     *
     * @param previousSessionId The previous session ID.
     * @param sessionId The session ID.
     */

     */
    public static class TestComponent {
        private static AtomicInteger instanceCounter = new AtomicInteger(0);
        private final int id;
        private boolean destroyed = false;

        public TestComponent() {
            this.id = instanceCounter.incrementAndGet();
        }

        public int getId() {
            return id;
        }

        public boolean isDestroyed() {

     * @param useRobotsTxtDisallows True to use disallow rules, false otherwise
     */
    public void setUseRobotsTxtDisallows(final boolean useRobotsTxtDisallows) {
        this.useRobotsTxtDisallows = useRobotsTxtDisallows;
    }

    /**
     * Sets whether to use robots.txt allow rules.
     *
     * @param useRobotsTxtAllows True to use allow rules, false otherwise
     */

     * @return true if the URL has been visited, false otherwise.
     */
    @Override
    public boolean visited(final OpenSearchUrlQueue urlQueue) {
        final String url = urlQueue.getUrl();
        if (StringUtil.isBlank(url)) {
            if (logger.isDebugEnabled()) {
                logger.debug("URL is a blank: {}", url);
            }
            return false;
        }

            server.stop();
        }
    }

    public void test_processRobotsTxt_disabled() {
        final String url = "http://localhost:7070/hoge.html";
        httpClient.robotsTxtHelper.setEnabled(false);
        httpClient.processRobotsTxt(url);
        assertTrue(true);
    }

    public void test_convertRobotsTxtPathPattern() {
        assertEquals("/.*\\?.*", httpClient.convertRobotsTxtPatternToRegex("/*?*"));

    }

    public void testParse_disable() {
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        robotsTxtHelper.setEnabled(false);
        try {
            assertNull(robotsTxtHelper.parse(in));
        } finally {
            robotsTxtHelper.setEnabled(true);
            CloseableUtil.closeQuietly(in);
        }
    }

    }

    @Override
    public String getRealm() {
        return null;
    }

    @Override
    public boolean isConnectionBased() {
        return false;
    }

    @Override
    public boolean isComplete() {
        return false;
    }

    @Override
    public Header authenticate(final Credentials credentials, final HttpRequest request) throws AuthenticationException {
        return null;

        CloseableUtil.closeQuietly(in);
        logger.info(content);
        assertTrue(content.contains("テスト"));
        assertFalse(content.contains("コメント"));
        xmlExtractor.setIgnoreCommentTag(false);
    }

    public void test_getXml_sjis() {
        final InputStream in = ResourceUtil.getResourceAsStream("extractor/test_sjis.xml");
        final String content = xmlExtractor.getText(in, null).getContent();

import org.mortbay.log.Log;

/**
 * @author shinsuke
 *
 */
public class CrawlerWebServer {
    private final File docRoot;

    private final Server server;

    private boolean tempDocRoot = false;

    public CrawlerWebServer(final int port) {
        this(port, createDocRoot(3));
        tempDocRoot = true;
    }

    public CrawlerWebServer(final int port, final File docRoot) {