dataCrawlingThreadList.add(dataCrawlingThread);
            dataCrawlingThreadStatusList.add(Constants.READY);

        }

        int startedCrawlerNum = 0;
        int activeCrawlerNum = 0;
        while (startedCrawlerNum < dataCrawlingThreadList.size()) {
            // Force to stop crawl
            if (systemHelper.isForceStop()) {
                for (final DataCrawlingThread crawlerThread : dataCrawlingThreadList) {

        }
        esCb.request().build(builder);
        SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
        String scrollId = response.getScrollId();
        try {
            while (scrollId != null) {
                final SearchHits searchHits = getSearchHits(response);
                final SearchHit[] hits = searchHits.getHits();
                if (hits.length == 0) {
                    break;

        if (text == null || fromText == null || toText == null) {
            return null;
        }
        final StringBuilder buf = new StringBuilder(100);
        int pos = 0;
        int pos2 = 0;
        while (true) {
            pos = text.indexOf(fromText, pos2);
            if (pos == 0) {
                buf.append(toText);
                pos2 = fromText.length();
            } else if (pos > 0) {

        final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
        int count = 1;
        String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
        while (StringUtil.isNotBlank(headerName)) {
            final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");

        }
        fail("could not start job.");
    }

    protected static void waitJob(final String namePrefix) {
        Boolean isRunning = false;
        int count = 0;

        while (count < 300 && !isRunning) { // Wait until the crawler starts
            ThreadUtil.sleep(500);
            count++;
            final Map<String, Object> scheduler = getSchedulerItem(namePrefix);

        br.addElement("Boundary size is limited by Framework.");
        br.addElement("Too long boundary is treated as 404 because it's thought of as attack.");
        br.addElement("");
        br.addElement("While, you can override the boundary limit size");
        br.addElement(" in " + FessMultipartRequestHandler.class.getSimpleName() + ".");
        br.addItem("Content Type");
        br.addElement(contentType);

            long updateTime = systemHelper.getCurrentTimeAsLong();
            int errorCount = 0;
            int emptyListCount = 0;
            long cleanupTime = -1;
            while (!finishCrawling || !accessResultList.isEmpty()) {
                try {
                    final int sessionIdListSize = finishedSessionIdList.size();
                    intervalControlHelper.setCrawlerRunning(true);

            }
            final Matcher matcher = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE).matcher(value);
            final StringBuffer buf = new StringBuffer(value.length() + 100);
            while (matcher.find()) {
                matcher.appendReplacement(buf, Matcher.quoteReplacement(highlightTagPre + matcher.group(0) + highlightTagPost));
            }
            matcher.appendTail(buf);

        try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, Constants.UTF_8))) {
            long id = 0;
            String line = null;
            while ((line = reader.readLine()) != null) {
                // Remove comments
                final String replacedLine = line.replaceAll("#.*$", StringUtil.EMPTY).trim();

                // Skip empty lines or comment lines

        final String repoContent = getRepositoryContent(url);
        final Matcher matcher = Pattern.compile("href=\"[^\"]*(" + artifactType.getId() + "[a-zA-Z0-9\\-]+)/?\"").matcher(repoContent);
        while (matcher.find()) {
            final String name = matcher.group(1);
            final String pluginUrl = url + (url.endsWith("/") ? name + "/" : "/" + name + "/");
            try {