if (childUrlSet != null) {
            // add an url
            try {
                storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
                        urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
            } catch (final Throwable t) {
                if (!ComponentUtil.available()) {

                    }
                }

            });
        } catch (final Exception e) {
            final CrawlingAccessException rcae = new CrawlingAccessException("Could not get a text from " + responseData.getUrl(), e);
            rcae.setLogLevel(CrawlingAccessException.WARN);
            throw rcae;
        }
        if (content == null) {
            content = StringUtil.EMPTY;
        }

        super(message, e);
        this.url = url;
        this.abort = abort;
    }

    public String getUrl() {
        return url;
    }

    public boolean aborted() {
        return abort;
    }

            if (pattern != null) {
                if (!pattern.matcher(failureUrl.getErrorName()).matches()) {
                    urlList.add(failureUrl.getUrl());
                }
            } else {
                urlList.add(failureUrl.getUrl());
            }
        }
        return urlList;
    }

    public OptionalEntity<CrawlingConfig> getDefaultConfig(final ConfigType configType) {

        setupEpg(_epgMap, et -> ((FailureUrl) et).getThreadName(), (et, vl) -> ((FailureUrl) et).setThreadName(DfTypeUtil.toString(vl)),
                "threadName");
        setupEpg(_epgMap, et -> ((FailureUrl) et).getUrl(), (et, vl) -> ((FailureUrl) et).setUrl(DfTypeUtil.toString(vl)), "url");
    }

    @Override
    public PropertyGateway findPropertyGateway(final String prop) {
        return doFindEpg(_epgMap, prop);
    }

        for (final EsAccessResult accessResult : arList) {
            if (logger.isDebugEnabled()) {
                logger.debug("Indexing {}", accessResult.getUrl());
            }
            accessResult.setStatus(Constants.DONE_STATUS);
            accessResultList.add(accessResult);

            if (accessResult.getHttpStatusCode() != 200) {

            break;
        }
    }

    protected void install(final Artifact artifact) {
        final String fileName = artifact.getFileName();
        final String url = artifact.getUrl();
        if (StringUtil.isBlank(url)) {
            throw new PluginException("url is blank: " + artifact.getName());
        }
        if (url.startsWith("http:") || url.startsWith("https:")) {

                    }
                } catch (final ChildUrlsException e) {
                    crawlerStatsHelper.record(keyObj, StatsAction.CHILD_URLS);
                    e.getChildUrlList().stream().map(RequestData::getUrl).forEach(urlQueue::offer);
                } catch (final DataStoreCrawlingException e) {
                    crawlerStatsHelper.record(keyObj, StatsAction.ACCESS_EXCEPTION);

                && isValidCanonicalUrl(responseData.getUrl(), canonicalUrl)) {
            final Set<RequestData> childUrlSet = new HashSet<>();
            childUrlSet.add(RequestDataBuilder.newRequestData().get().url(canonicalUrl).build());
            logger.info("CANONICAL: {} -> {}", responseData.getUrl(), canonicalUrl);

            final Throwable e) {

        final CrawlingConfig crawlingConfig = getCrawlingConfig(crawlerContext.getSessionId());
        final String url = urlQueue.getUrl();

        final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
        failureUrlService.store(crawlingConfig, errorName, url, e);
    }