}
                final int depth = urlQueue.getDepth() == null ? 1 : urlQueue.getDepth() + 1;
                if (crawlerContext.getMaxDepth() < 0 || depth <= crawlerContext.getMaxDepth()) {
                    // add and filter urls
                    storeChildUrls(crawlerContext, resultData.getChildUrlSet(), urlQueue.getUrl(), depth, resultData.getEncoding());

                                fessConfig.getIndexFieldClickCount(), fessConfig.getIndexFieldFavoriteCount() });
                if (document == null) {
                    storeChildUrlsToQueue(urlQueue, getChildUrlSet(searchEngineClient, id));
                    return true;
                }

                final Date expires = DocumentUtil.getValue(document, fessConfig.getIndexFieldExpires(), Date.class);

                    logger.info("META(robots=noindex): {}", responseData.getUrl());
                    storeChildUrls(responseData, resultData);
                    throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots");
                }
                if (nofollow) {
                    logger.info("META(robots=nofollow): {}", responseData.getUrl());
                    responseData.setNoFollow(true);