// UrlQueue<?> urlQueue = (UrlQueue<?>) objs[1];
        final ResponseData responseData = (ResponseData) objs[2];
        if (logger.isDebugEnabled()) {
            logger.debug("No rule for ({}, {}). PLEASE CHECK YOUR CONFIGURATION.", responseData.getUrl(), responseData.getMimeType());
        }
    }

    protected void processNoResponseProcessor(final Object... objs) {

    //                                                                              ======
    @Override
    protected int delegateSelectCountUniquely(final ConditionBean cb) {
        // #pending check response and cast problem
        final SearchRequestBuilder builder = client.prepareSearch(asEsIndex());
        final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
        if (esCb.getPreference() != null) {

     * StringUtils.isAsciiPrintable("\u007f") = false
     * StringUtils.isAsciiPrintable("Ceki G\u00fclc\u00fc") = false
     * </pre>
     *
     * @param cs the CharSequence to check, may be null
     * @return {@code true} if every character is in the range 32 thru 126
     */
    public static boolean isAsciiPrintable(final CharSequence cs) {
        if (cs == null) {
            return false;

                responseData.setCharSet(charset);
                responseData.setContentLength(0);
            } else if (file.isFile()) {
                // check file size
                responseData.setContentLength(file.length());
                checkMaxContentLength(responseData);

                parseFileOwnerAttribute(responseData, file);

        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        final CrawlerStatsHelper crawlerStatsHelper = ComponentUtil.getCrawlerStatsHelper();
        synchronized (indexUpdateCallback) {
            // required check
            if (!dataMap.containsKey(fessConfig.getIndexFieldUrl()) || dataMap.get(fessConfig.getIndexFieldUrl()) == null) {
                logger.warn("Could not add a doc. Invalid data: {}", dataMap);
                return;

                startedCrawlerNum++;
                activeCrawlerNum++;
                ThreadUtil.sleep(crawlingExecutionInterval);
                continue;
            }

            // check status
            for (int i = 0; i < startedCrawlerNum; i++) {
                if (!dataCrawlingThreadList.get(i).isRunning() && Constants.RUNNING.equals(dataCrawlingThreadStatusList.get(i))) {

            responseData.setHttpStatusCode(Constants.OK_STATUS_CODE);
            responseData.setCharSet(Constants.UTF_8);
            responseData.setLastModified(file.getTimestamp().getTime());

            // check file size
            responseData.setContentLength(file.getSize());
            checkMaxContentLength(responseData);

            if (file.getUser() != null) {

                    startedCrawlerNum++;
                    activeCrawlerNum++;
                    ThreadUtil.sleep(crawlingExecutionInterval);
                    continue;
                }

                // check status
                for (int i = 0; i < startedCrawlerNum; i++) {
                    if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE

    }

    /**
     * Returns data as HTML content of String.
     *
     */
    @Override
    public Object getData(final AccessResultData<?> accessResultData) {
        // check transformer name
        if (!getName().equals(accessResultData.getTransformerName())) {
            throw new CrawlerSystemException(

        if (idx >= 0) {
            hostUrl = url.substring(0, idx);
        } else {
            hostUrl = url;
        }
        final String robotTxtUrl = hostUrl + "/robots.txt";

        // check url
        if (crawlerContext.getRobotsTxtUrlSet().contains(robotTxtUrl)) {
            if (logger.isDebugEnabled()) {
                logger.debug("{} is already visited.", robotTxtUrl);
            }