* @author shinsuke
 *
 */
public class CrawlerWebServer {
    private int port = 8080;

    private File docRoot;

    private Server server;

    private boolean tempDocRoot = false;

    public CrawlerWebServer(final int port) {
        this(port, createDocRoot(3));
        tempDocRoot = true;
    }

    public CrawlerWebServer(final int port, final File docRoot) {
        this.port = port;

    /** Constant representing the pre-processing state. */
    int PRE_PROCESSING = 1;

    /** Constant representing the post-processing state. */
    int POST_PROCESSING = 2;

    /** Constant indicating that there are no URLs in the queue. */
    int NO_URL_IN_QUEUE = 4;

    /** Constant indicating that the crawler is waiting for new URLs. */
    int WAIT_NEW_URL = 8;

    /**

        return tempFile;
    }

    /*
     * (non-Javadoc)
     *
     * @see java.io.InputStream#read()
     */
    @Override
    public int read() throws IOException {
        return fileInputStream.read();
    }

    @Override
    public int available() throws IOException {
        return fileInputStream.available();
    }

    /**

        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final int maxCount = 50;
            final int numOfThread = 10;

            final File file = File.createTempFile("crawler-", "");
            file.delete();
            file.mkdirs();
            file.deleteOnExit();

                    || PRIORITY_ELEMENT.equals(qName)) {
                buf = new StringBuilder();
            }
        }

        @Override
        public void characters(final char[] ch, final int offset, final int length) {
            if (buf != null) {
                buf.append(new String(ch, offset, length));
            }
        }

        @Override

     */
    protected int memorySize = 1024 * 1024; //1mb

    /**
     * Max size of an alpha-numeric term.
     */
    protected int maxAlphanumTermSize = -1;

    /**
     * Max size of a symbol term.
     */
    protected int maxSymbolTermSize = -1;

    /**
     * Tika config.
     */
    protected TikaConfig tikaConfig;

    public void test_isSuccessful() {
        DefaultResponseProcessor processor = new DefaultResponseProcessor();
        processor.setSuccessfulHttpCodes(new int[] { 200 });
        processor.setNotModifiedHttpCodes(new int[] { 304 });

        ResponseData responseData = new ResponseData();
        responseData.setHttpStatusCode(200);
        assertTrue(processor.isSuccessful(responseData));

    /**
     * Number of shards for the queue index.
     */
    protected int queueShards = Runtime.getRuntime().availableProcessors() * 2;

    /**
     * Number of shards for the data index.
     */
    protected int dataShards = Runtime.getRuntime().availableProcessors() * 2;

    /**
     * Number of shards for the filter index.
     */
    protected int filterShards = Runtime.getRuntime().availableProcessors() * 2;

    /**

 * @author shinsuke
 *
 */
public class CrawlerWebServer {
    private int port = 8080;

    private File docRoot;

    private Server server;

    private boolean tempDocRoot = false;

    public CrawlerWebServer(final int port) {
        this(port, createDocRoot(3));
        tempDocRoot = true;
    }

    public CrawlerWebServer(final int port, final File docRoot) {
        this.port = port;

        assertEquals("123# !$", normalizeText(" 123#123!$123  ", 100, -1, -1, true));
    }

    public static String normalizeText(final String str, final int initialCapacity, final int maxAlphanumTermSize,
            final int maxSymbolTermSize, final boolean removeDuplication) {
        if (str == null) {
            return StringUtil.EMPTY;
        }
        try (final Reader reader = new StringReader(str)) {