return noFollow;
    }

    public void addMetaData(final String name, final Object value) {
        metaDataMap.put(name, value);
    }

    public Map<String, Object> getMetaDataMap() {
        return metaDataMap;
    }

    public void addChildUrl(final RequestData url) {
        childUrlSet.add(url);
    }

    public void removeChildUrl(final RequestData url) {

                storeChildUrls(responseData, resultData);
            }
        } finally {
            xpathAPI.remove();
        }

        final Object redirectUrlObj = responseData.getMetaDataMap().get(LOCATION_HEADER);
        if (redirectUrlObj instanceof String) {
            final UrlConvertHelper urlConvertHelper = crawlerContainer.getComponent("urlConvertHelper");

                return;
            }
        } else if (Boolean.parseBoolean(ignore)) {
            return;
        }

        // X-Robots-Tag
        responseData.getMetaDataMap().entrySet().stream().filter(e -> X_ROBOTS_TAG.equalsIgnoreCase(e.getKey()) && e.getValue() != null)
                .forEach(e -> {
                    boolean noindex = false;
                    boolean nofollow = false;