return tempDir;
        } catch (final Exception e) {
            throw new CrawlerSystemException(e);
        }
    }

    private static void generateContents(final File dir, final int count) throws Exception {
        if (count <= 0) {
            return;
        }

        final String content = getHtmlContent(count);

        final File indexFile = new File(dir, "index.html");

     * @param threadPool The thread pool.
     */
    public SuggestIndexer(final Client client, final String index, final ReadingConverter readingConverter,
            final ReadingConverter contentsReadingConverter, final Normalizer normalizer, final SuggestAnalyzer analyzer,
            final SuggestSettings settings, final ExecutorService threadPool) {
        this.client = client;
        this.index = index;

        final ThreadSafeRuleManager threadSafeManager = new ThreadSafeRuleManager();
        final int threadCount = 10;
        final int rulesPerThread = 100;
        final CountDownLatch startLatch = new CountDownLatch(1);
        final CountDownLatch endLatch = new CountDownLatch(threadCount);

        for (int i = 0; i < threadCount; i++) {
            final int threadId = i;

    @Override
    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {
            final BufferedInputStream bis = new BufferedInputStream(in);
            final String enc = getEncoding(bis);

                }
            }
        } catch (final CrawlingAccessException e) {
            throw e;
        } catch (final Exception e) {
            throw new ExtractException("Failed to extract data.", e);
        } finally {
            if (dfos != null && !dfos.isInMemory()) {
                final File file = dfos.getFile();
                try {

    /** Property name for proxy authentication scheme setting */
    public static final String PROXY_AUTH_SCHEME_PROPERTY = "proxyAuthScheme";

    /** Property name for proxy credentials setting */
    public static final String PROXY_CREDENTIALS_PROPERTY = "proxyCredentials";

    /** Property name for user agent setting */

     */
    @Override
    protected boolean exists(final String sessionId, final String url) {
        final boolean ret = super.exists(sessionId, url);
        if (!ret) {
            final QueueHolder queueHolder = getQueueHolder(sessionId);
            final Queue<OpenSearchUrlQueue> waitingQueue = queueHolder.waitingQueue;
            final Queue<OpenSearchUrlQueue> crawlingQueue = queueHolder.crawlingQueue;

    private final ThreadLocal<XPathAPI> xpathAPI = new ThreadLocal<>();

    /**
     * Creates a new HtmlExtractor instance.
     */
    public HtmlExtractor() {
        super();
    }

    @Override
    protected ExtractData createExtractData(final String content) {
        final DOMParser parser = getDomParser();
        try (final Reader reader = new StringReader(content)) {

            final Header[] headers = response.getAllHeaders();
            for (final Header header : headers) {
                data.putValue(header.getName(), header.getValue());
            }
        } catch (final IOException e) {
            throw new ExtractException(e);
        } finally {
            if (accessTimeout != null) {

    private static final Logger logger = LogManager.getLogger(FileSystemClient.class);

    /** Key for file attribute view in metadata */
    public static final String FILE_ATTRIBUTE_VIEW = "fileAttributeView";

    /** Key for file user in metadata */
    public static final String FS_FILE_USER = "fsFileUser";

    /** Key for file groups in metadata */
    public static final String FS_FILE_GROUPS = "fsFileGroups";