if (childUrlSet != null) {
            // add an url
            try {
                storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
                        urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
            } catch (final Throwable t) {
                if (!ComponentUtil.available()) {

                    }
                }

            });
        } catch (final Exception e) {
            final CrawlingAccessException rcae = new CrawlingAccessException("Could not get a text from " + responseData.getUrl(), e);
            rcae.setLogLevel(CrawlingAccessException.WARN);
            throw rcae;
        }
        if (content == null) {
            content = StringUtil.EMPTY;
        }

        /**
         * @return URL
         */
        public URL getURL() {
            return url_;
        }

        /**
         * @param url
         */
        public void setURL(final URL url) {
            url_ = url;
        }

        /**
         * @return Returns the cal.
         */
        public Calendar getCal() {
            return cal;
        }

        /**

        for (final EsAccessResult accessResult : arList) {
            if (logger.isDebugEnabled()) {
                logger.debug("Indexing {}", accessResult.getUrl());
            }
            accessResult.setStatus(Constants.DONE_STATUS);
            accessResultList.add(accessResult);

            if (accessResult.getHttpStatusCode() != 200) {

        public String pathOfLocalRepositoryMetadata(ArtifactMetadata metadata, ArtifactRepository repository) {
            return null;
        }

        public String getUrl() {
            return repository.getUrl();
        }

        public void setUrl(String url) {}

        public String getBasedir() {
            return null;
        }

        public String getProtocol() {

                    }
                } catch (final ChildUrlsException e) {
                    crawlerStatsHelper.record(keyObj, StatsAction.CHILD_URLS);
                    e.getChildUrlList().stream().map(RequestData::getUrl).forEach(urlQueue::offer);
                } catch (final DataStoreCrawlingException e) {
                    crawlerStatsHelper.record(keyObj, StatsAction.ACCESS_EXCEPTION);

        if (r1 == r2) {
            return true;
        }

        return Objects.equals(r1.getId(), r2.getId())
                && Objects.equals(r1.getUrl(), r2.getUrl())
                && repositoryPolicyEquals(r1.getReleases(), r2.getReleases())
                && repositoryPolicyEquals(r1.getSnapshots(), r2.getSnapshots());
    }

            if (pattern != null) {
                if (!pattern.matcher(failureUrl.getErrorName()).matches()) {
                    urlList.add(failureUrl.getUrl());
                }
            } else {
                urlList.add(failureUrl.getUrl());
            }
        }
        return urlList;
    }

    public OptionalEntity<CrawlingConfig> getDefaultConfig(final ConfigType configType) {

        repo.id(settingsRepo.getId());
        repo.layout(settingsRepo.getLayout());
        repo.name(settingsRepo.getName());
        repo.url(settingsRepo.getUrl());

        repo.location("id", toLocation(settingsRepo.getLocation("id")));
        repo.location("layout", toLocation(settingsRepo.getLocation("layout")));

        urlList = fessXpathTransformer.convertChildUrlList(urlList);
        assertEquals(2, urlList.size());
        assertEquals("http://www.example.com", urlList.get(0).getUrl());
        assertEquals("http://www.test.com", urlList.get(1).getUrl());

        urlList.clear();
        urlList.add(RequestDataBuilder.newRequestData().get().url("feed://www.example.com").build());