storeChildUrls(responseData, resultData);
            }
        } finally {
            xpathAPI.remove();
        }

        final Object redirectUrlObj = responseData.getMetaDataMap().get(LOCATION_HEADER);
        if (redirectUrlObj instanceof String) {
            final UrlConvertHelper urlConvertHelper = crawlerContainer.getComponent("urlConvertHelper");
            final String redirectUrl;

                return;
            }
        } else if (Boolean.parseBoolean(ignore)) {
            return;
        }

        // X-Robots-Tag
        responseData.getMetaDataMap()
                .entrySet()
                .stream()
                .filter(e -> X_ROBOTS_TAG.equalsIgnoreCase(e.getKey()) && e.getValue() != null)
                .forEach(e -> {