/**
 * @author shinsuke
 *
 */
public class CrawlerWebServer {
    private int port = 8080;

    private File docRoot;

    private Server server;

    private boolean tempDocRoot = false;

    public CrawlerWebServer(final int port) {
        this(port, createDocRoot(3));
        tempDocRoot = true;
    }

    public CrawlerWebServer(final int port, final File docRoot) {
        this.port = port;

        nBestExamples = settings.get(NBEST_EXAMPLES);
    }

    private static String parse(String rule, Set<String> dup) {
        String[] values = CSVUtil.parse(rule);
        if (dup.add(values[0]) == false) {
            throw new IllegalArgumentException("Found duplicate term [" + values[0] + "] in user dictionary.");
        }
        return rule;
    }

     */
    void saveSession(String sessionId);

    /**
     * Checks if a URL has been visited.
     *
     * @param urlQueue The URL queue.
     * @return true if the URL has been visited, otherwise false.
     */
    boolean visited(QUEUE urlQueue);

    /**
     * Generates URL queues.
     *
     * @param previousSessionId The previous session ID.
     * @param sessionId The session ID.
     */

     */
    public static class TestComponent {
        private static AtomicInteger instanceCounter = new AtomicInteger(0);
        private final int id;
        private boolean destroyed = false;

        public TestComponent() {
            this.id = instanceCounter.incrementAndGet();
        }

        public int getId() {
            return id;
        }

        public boolean isDestroyed() {

     * @param useRobotsTxtDisallows True to use disallow rules, false otherwise
     */
    public void setUseRobotsTxtDisallows(final boolean useRobotsTxtDisallows) {
        this.useRobotsTxtDisallows = useRobotsTxtDisallows;
    }

    /**
     * Sets whether to use robots.txt allow rules.
     *
     * @param useRobotsTxtAllows True to use allow rules, false otherwise
     */

        fileTransformer = new FileTransformer();
        fileTransformer.setName("fileTransformer");
        Map<String, String> featureMap = newHashMap();
        featureMap.put("http://xml.org/sax/features/namespaces", "false");
        fileTransformer.setFeatureMap(featureMap);
        Map<String, String> propertyMap = newHashMap();
        fileTransformer.setPropertyMap(propertyMap);
        Map<String, String> childUrlRuleMap = newHashMap();

     * @return true if the URL has been visited, false otherwise.
     */
    @Override
    public boolean visited(final OpenSearchUrlQueue urlQueue) {
        final String url = urlQueue.getUrl();
        if (StringUtil.isBlank(url)) {
            if (logger.isDebugEnabled()) {
                logger.debug("URL is a blank: {}", url);
            }
            return false;
        }

    }

    public void testParse_disable() {
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        robotsTxtHelper.setEnabled(false);
        try {
            assertNull(robotsTxtHelper.parse(in));
        } finally {
            robotsTxtHelper.setEnabled(true);
            CloseableUtil.closeQuietly(in);
        }
    }

			</plugin>
		</plugins>
	</build>
	<repositories>
		<repository>
			<id>snapshots.central.sonatype.com</id>
			<url>https://central.sonatype.com/repository/maven-snapshots</url>
			<releases>
				<enabled>false</enabled>
			</releases>
			<snapshots>
				<enabled>true</enabled>
			</snapshots>
		</repository>
	</repositories>
	<dependencies>
		<dependency>
			<groupId>com.ibm.icu</groupId>

            server.stop();
        }
    }

    public void test_processRobotsTxt_disabled() {
        final String url = "http://localhost:7070/hoge.html";
        httpClient.robotsTxtHelper.setEnabled(false);
        httpClient.processRobotsTxt(url);
        assertTrue(true);
    }

    public void test_convertRobotsTxtPathPattern() {
        assertEquals("/.*\\?.*", httpClient.convertRobotsTxtPatternToRegex("/*?*"));