totalDocNum = getTotal();
    }

    @Override
    public synchronized Map<String, Object> read() {
        while (!isFinished.get() && queue.isEmpty()) {
            addDocumentToQueue();
        }

        return queue.poll();
    }

    @Override
    public void close() {
        isFinished.set(true);
        queue.clear();
    }

    }

    @Override
    public String getContentType(final InputStream is, final Map<String, String> params) {
        final String filename = params.get(ExtractData.RESOURCE_NAME_KEY);
        if (StringUtil.isEmpty(filename) && is == null) {
            throw new MimeTypeException("The filename and input stream is empty.");
        }

        final Metadata metadata = new Metadata();

            List<String> ruleList =
                    Analysis.parseWordList(env, settings, USER_DICT_PATH_OPTION, USER_DICT_RULES_OPTION, s -> parse(s, dup));
            if (ruleList == null || ruleList.isEmpty()) {
                return null;
            }

            StringBuilder sb = new StringBuilder();
            for (String line : ruleList) {
                sb.append(line).append(System.lineSeparator());

        }

        // Wait for all threads to complete
        for (Thread thread : threads) {
            thread.join();
        }

        // Verify results
        assertTrue(exceptions.isEmpty());
        assertEquals(threadCount * objectsPerThread, createdComponents.size());

        // Check all components are unique (for prototype)
        for (int i = 0; i < createdComponents.size(); i++) {

                    ThreadUtil.sleep(100L);
                }
                throw new ExtractException("PDFBox process cannot finish in " + timeout + " sec.");
            }
            if (!exceptionSet.isEmpty()) {
                throw exceptionSet.iterator().next();
            }
            writer.flush();
            final ExtractData extractData = new ExtractData(writer.toString());

                        }
                    }
                }

                wordsList.add(word);
                readingList.add(l.toArray(new String[l.size()]));
            }
            if (wordsList.isEmpty()) {
                return null;
            }
            return new SuggestItem(wordsList.toArray(new String[wordsList.size()]), readingList.toArray(new String[readingList.size()][]),

     * @param client The CrawlerClient instance.
     */
    public void addClient(final List<String> regexList, final CrawlerClient client) {
        if (regexList == null || regexList.isEmpty()) {
            throw new CrawlerSystemException("A regular expression list is null or empty.");
        }
        if (client == null) {
            throw new CrawlerSystemException("CrawlerClient is null.");
        }

     * @param uri the URI to preprocess
     * @return the preprocessed URI
     * @throws CrawlerSystemException if the URI is empty
     */
    protected String preprocessUri(final String uri) {
        if (StringUtil.isEmpty(uri)) {
            throw new CrawlerSystemException("The uri is empty.");
        }

        return uri;
    }

    /**
     * Returns the character set for the given SMB1 file.
     *

     */
    public void addExtractor(final List<String> keyList, final Extractor extractor) {
        if (keyList == null || keyList.isEmpty()) {
            throw new CrawlerSystemException("The key list is empty.");
        }
        keyList.stream().distinct().forEach(key -> addExtractor(key, extractor));
    }

    /**

    public void test_robotsTxtUrlSet() {
        Set<String> urlSet = crawlerContext.getRobotsTxtUrlSet();
        assertNotNull(urlSet);
        assertTrue(urlSet instanceof LruHashSet);
        assertTrue(urlSet.isEmpty());

        // Add URLs to default set
        urlSet.add("http://example.com/robots.txt");
        urlSet.add("http://test.com/robots.txt");
        assertEquals(2, crawlerContext.getRobotsTxtUrlSet().size());