&& isValidCanonicalUrl(responseData.getUrl(), canonicalUrl)) {
            final Set<RequestData> childUrlSet = new HashSet<>();
            childUrlSet.add(RequestDataBuilder.newRequestData().get().url(canonicalUrl).build());
            logger.info("Canonical URL redirect: from={}, to={}", responseData.getUrl(), canonicalUrl);

            assertEquals("application/octet-stream", responseData.getMimeType());
            assertNull(responseData.getResponseBody());
            assertNull(responseData.getMetaDataMap().get("label"));
        }
        try (final ResponseData responseData = s3Client.doHead("s3://fess/dir1/dir2/file3.txt")) {
            assertEquals("s3://fess/dir1/dir2/file3.txt", responseData.getUrl());

                    }
                    break;
                default:
                    logger.error("Unknown thumbnail result: {} -> {}", thumbnailId, responseData.getUrl());
                    break;
                }
            } catch (final Throwable t) {
                logger.warn("Failed to create thumbnail: {} -> {} ({}:{})", thumbnailId, responseData.getUrl(),

                    if (logger.isDebugEnabled()) {
                        logger.debug("Skipping crawl request for url='{}' at depth={} (maxDepth={}, alreadyProcessed={})",
                                crawlRequest.getUrl(), crawlRequest.getDepth(), maxDepth, processedUrls.contains(crawlRequest.getUrl()));
                    }
                    continue;

                    final SmbFile[] files = file.listFiles();
                    if (files != null) {
                        for (final SmbFile f : files) {
                            final String childUri = f.getURL().toExternalForm();
                            requestDataSet.add(RequestDataBuilder.newRequestData().get().url(childUri).build());
                        }
                    }
                }

            assertEquals("application/octet-stream", responseData.getMimeType());
            assertNull(responseData.getResponseBody());
            assertNull(responseData.getMetaDataMap().get("label"));
        }
        try (final ResponseData responseData = gcsClient.doHead("gcs://fess/dir1/dir2/file3.txt")) {
            assertEquals("gcs://fess/dir1/dir2/file3.txt", responseData.getUrl());

                                + " (" + repository.getUrl() + ")");
            }

            return false;
        }

        if (getLogger().isDebugEnabled()) {
            getLogger()
                    .debug("Determining update check for " + artifact + " (" + file + ") from " + repository.getId()
                            + " (" + repository.getUrl() + ")");
        }

        if (file == null) {

                }
                return;
            }
            logger.warn("Failed to store failure url: url={}", urlQueue.getUrl(), e);
        }

        if (cae.isDebugEnabled()) {
            logger.debug("[{}] Crawling Access Exception: url={}", failureUrlId, urlQueue.getUrl(), cae);
        } else if (cae.isInfoEnabled()) {
            logger.info("[{}] {}", failureUrlId, cae.getMessage());

    @Override
    public ResultData transform(final ResponseData responseData) {
        if (responseData == null || !responseData.hasResponseBody()) {
            final String url = responseData != null ? responseData.getUrl() : "unknown";
            throw new CrawlingAccessException("No response body for URL: " + url + ". Cannot transform empty response.");
        }

        final ResultData resultData = new ResultData();

        if (childUrlSet != null) {
            // add an url
            try {
                storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
                        urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
            } catch (final Throwable t) {
                if (!ComponentUtil.available()) {