}
    }

    /**
     * Returns a string representation of this object.
     * @return A string representation.
     */
    @Override
    public String toString() {
        return "RobotsTxt [directiveMap=" + directiveMap + ", sitemapList=" + sitemapList + "]";
    }

                logger.error("Failed to access {}, code: {}.", url, response.getStatusLine().getStatusCode());
                return null;
            }

            data.setContent(EntityUtils.toString(response.getEntity(), Charsets.UTF_8));
            final Header[] headers = response.getAllHeaders();
            for (final Header header : headers) {
                data.putValue(header.getName(), header.getValue());

        converter.init();
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < 1000; i++) {
            sb.append("word").append(i).append(" ");
        }
        String text = sb.toString();
        String field = "content";

        List<String> readings = converter.convert(text, field, "en");

        assertNotNull(readings);
        assertEquals(1, readings.size());

            }
            if (!exceptionSet.isEmpty()) {
                throw exceptionSet.iterator().next();
            }
            writer.flush();
            final ExtractData extractData = new ExtractData(writer.toString());
            extractMetadata(document, extractData);
            return extractData;
        } catch (final Exception e) {
            throw new ExtractException(e);
        }
    }

    /**

                    }
                }
            }
        } catch (final UnsupportedEncodingException e) {
            return filePath;
        }
        return buf.toString();
    }

    /**
     * Gets the character set for the given file.
     *
     * @param file the file to get the character set for
     * @return the character set
     */

            crawlerThread.setClientFactory(clientFactory);
            threads[i] =
                    new Thread(crawlerThreadGroup, crawlerThread, "Crawler-" + crawlerContext.sessionId + "-" + Integer.toString(i + 1));
            threads[i].setDaemon(daemon);
            threads[i].setPriority(threadPriority);
        }

        // run
        crawlerContext.setStatus(CrawlerStatus.RUNNING);

            final SearchHit hit = hits[i];

            final Map<String, Object> source = hit.getSourceAsMap();
            final String text = source.get(FieldNames.TEXT).toString();
            if (skipDuplicateWords) {
                final String duplicateCheckStr = text.replace(" ", "");

                buf.append(e.getMessage());
                if (e.getCause() != null) {
                    buf.append(e.getCause().getMessage());
                }
                logger.info(buf.toString());
            } else if (logger.isDebugEnabled()) {
                logger.debug("Crawling Access Exception at {}", url, e);
            }
        }

        // request header

            logger.debug("Initializing SmbClient...");
        }

        super.init();

        final Properties props = new Properties();
        System.getProperties().entrySet().stream().filter(e -> e.getKey().toString().startsWith("jcifs.")).forEach(e -> {
            props.setProperty((String) e.getKey(), (String) e.getValue());
        });
        try {
            cifsContext = new BaseContext(new PropertyConfiguration(props));

            final int port = getPort();
            if (port != DEFAULT_FTP_PORT) {
                buf.append(':').append(port);
            }
            buf.append(path);
            final String url = normalize(buf.toString());
            if ("/".equals(path)) {
                return url;
            }
            return url.replaceAll("/+$", "");
        }

        /**