/**
     * Number of shards for the queue index.
     */
    protected int queueShards = Runtime.getRuntime().availableProcessors() * 2;

    /**
     * Number of shards for the data index.
     */
    protected int dataShards = Runtime.getRuntime().availableProcessors() * 2;

    /**
     * Number of shards for the filter index.
     */
    protected int filterShards = Runtime.getRuntime().availableProcessors() * 2;

    /**

     */
    public static final int OK_STATUS = 0;

    /**
     * The status code for Not Modified.
     */
    public static final int NOT_MODIFIED_STATUS = 304;

    /**
     * The HTTP status code for OK.
     */
    public static final int OK_STATUS_CODE = 200;

    /**
     * The HTTP status code for Not Modified.
     */
    public static final int NOT_MODIFIED_STATUS_CODE = 304;

    /**

        private final Reader reader;

        private int initialCapacity = 10000;

        private int maxAlphanumTermSize = -1;

        private int maxSymbolTermSize = -1;

        private boolean duplicateTermRemoved = false;

        /**
         * Array of space characters. Default includes common space characters.
         */
        private int[] spaceChars = { '\u0020', '\u00a0', '\u3000', '\ufffd' };

     * @return the total number of bytes read into the buffer, or -1 if there is no more data
     * @throws IOException if an I/O error occurs
     */
    @Override
    public int read(final byte[] b, final int off, final int len) throws IOException {
        return inputStream.read(b, off, len);
    }

    /**
     * Reads some number of bytes from the input stream and stores them into the buffer array b.
     *

        for (int i = 0; i < threadCount; i++) {
            final int threadId = i;
            executor.submit(new Runnable() {
                @Override
                public void run() {
                    try {
                        startLatch.await();
                        for (int j = 0; j < operationsPerThread; j++) {
                            // Read name

    private final File docRoot;

    private final Server server;

    private boolean tempDocRoot = false;

    public CrawlerWebServer(final int port) {
        this(port, createDocRoot(3));
        tempDocRoot = true;
    }

    public CrawlerWebServer(final int port, final File docRoot) {
        this.docRoot = docRoot;

        server = new Server(port);

                    processor.setTransformer(container.getComponent("fileTransformer"));
                    processor.setSuccessfulHttpCodes(new int[] { 200 });
                    processor.setNotModifiedHttpCodes(new int[] { 304 });
                })//
                .<RegexRule> singleton("fileRule", RegexRule.class, rule -> {
                    rule.setRuleId("fileRule");

    private final CharsRef scratchChars = new CharsRef();

    private int longestMatchEndOffset;

    private int ch;

    private final char[] readBuffer;

    private int readBufferIndex;

    private int readBufferLen;

    StringBuilder block;

    int blkStart;

    int nextBlkStart;

    private int finalOffset;

    private final PriorityQueue<MyToken> queue;

            String[] words = text.trim().split("[\\s@#$.,!?;:()\\[\\]{}\"']+");

            int position = 0;
            int offset = 0;
            for (String word : words) {
                if (!word.isEmpty()) {
                    int startOffset = text.indexOf(word, offset);
                    int endOffset = startOffset + word.length();

                    // Handle decimal numbers

        final CrawlerWebServer server = new CrawlerWebServer(7070);
        server.start();

        final String url = "http://localhost:7070/";
        try {
            final int maxCount = 50;
            final int numOfThread = 10;

            final File file = File.createTempFile("crawler-", "");
            file.delete();
            file.mkdirs();
            file.deleteOnExit();