crawlerContext.intervalController.delay(IntervalController.PRE_PROCESSING);
                        }

                        final boolean contentUpdated = isContentUpdated(client, urlQueue);

                        if (contentUpdated) {
                            log(logHelper, LogType.GET_CONTENT, crawlerContext, urlQueue);
                            // access an url

    protected static final String CRAWLER_CLIENTS = "crawlerClients";

    protected ConcurrentHashMap<String, Pair<String, Pattern>> clientRuleCache = new ConcurrentHashMap<>();

    @Override
    protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?> urlQueue) {
        if (ComponentUtil.getFessConfig().isIncrementalCrawling()) {

            final SystemHelper systemHelper = ComponentUtil.getSystemHelper();