/**
     * @throws Exception
     */
    public void testGetBytes() throws Exception {
        final InputStream is = ResourceUtil.getResourceAsStream(StringUtil.replace(getClass().getName(), ".", "/") + ".class");
        try {
            assertNotNull("1", InputStreamUtil.getBytes(is));
        } finally {
            is.close();
        }
    }

     * @throws CIFSException if an error occurs during initialization
     */
    public DirFileEntryEnumIterator2(final SmbTreeHandleImpl th, final SmbResource parent, final String wildcard,
            final ResourceNameFilter filter, final int searchAttributes) throws CIFSException {
        super(th, parent, wildcard, filter, searchAttributes);
    }

    /**
     * {@inheritDoc}
     *

     */
    // GET /api/admin/badword/settings
    // PUT /api/admin/badword/settings
    @Execute
    public JsonResponse<ApiResult> settings(final SearchBody body) {
        validateApi(body, messages -> {});
        final BadWordPager pager = copyBeanToNewBean(body, BadWordPager.class);
        final List<BadWord> list = badWordService.getBadWordList(pager);
        return asJson(

            return tempDir;
        } catch (final Exception e) {
            throw new CrawlerSystemException(e);
        }
    }

    private static void generateContents(final File dir, final int count) throws Exception {
        if (count <= 0) {
            return;
        }

        final String content = getHtmlContent(count);

        final File indexFile = new File(dir, "index.html");

        dataStore.alive = true;

        final int readerCount = 5;
        final int iterations = 1000;
        final Thread[] readers = new Thread[readerCount];
        final Exception[] exceptions = new Exception[readerCount + 1];
        final int[] trueCount = new int[readerCount];
        final int[] falseCount = new int[readerCount];

        // Start reader threads

     */
    protected void doCrawl(final String sessionId, final List<DataConfig> configList) {
        final int multiprocessCrawlingCount = ComponentUtil.getFessConfig().getCrawlingThreadCount();

        final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
        final long startTime = systemHelper.getCurrentTimeAsLong();

    }

    @Override
    protected int delegateUpdate(final Entity entity, final UpdateOption<? extends ConditionBean> option) {
        final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
        final IndexRequestBuilder builder = createUpdateRequest(esEntity);

        final IndexResponse response = builder.execute().actionGet(indexTimeout);
        final long seqNo = response.getSeqNo();

      assertExpectedBytes(readBytes);
    } finally {
      in.close();
    }
  }

  public void testOpenBufferedStream() throws IOException {
    InputStream in = source.openBufferedStream();
    try {
      byte[] readBytes = ByteStreams.toByteArray(in);
      assertExpectedBytes(readBytes);
    } finally {
      in.close();
    }
  }

 */
public final class Crawler {
  private final OkHttpClient client;
  private final Set<HttpUrl> fetchedUrls = Collections.synchronizedSet(new LinkedHashSet<>());
  private final BlockingQueue<HttpUrl> queue;
  private final ConcurrentHashMap<String, AtomicInteger> hostnames = new ConcurrentHashMap<>();
  private final int hostLimit;

    protected Hc4Authentication[] convertFromConfig(final WebAuthenticationConfig[] configs) {
        final List<Hc4Authentication> result = new ArrayList<>();
        for (final WebAuthenticationConfig config : configs) {
            try {
                final AuthScope authScope = new AuthScope(config.getHost(), config.getPort(), config.getRealm(), config.getScheme());