childUrlSet.add(url);
    }

    public void removeChildUrl(final RequestData url) {
        childUrlSet.remove(url);
    }

    public Set<RequestData> getChildUrlSet() {
        return childUrlSet;
    }

    public RequestData getRequestData() {
        final RequestData requestData = new RequestData();
        requestData.setMethod(method);

                    logger.info("META(robots=noindex): {}", responseData.getUrl());
                    storeChildUrls(responseData, resultData);
                    throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots");
                }
                if (nofollow) {
                    logger.info("META(robots=nofollow): {}", responseData.getUrl());
                    responseData.setNoFollow(true);

                }
                requestDataList = convertChildUrlList(requestDataList);
                resultData.addAllUrl(requestDataList);
            });

            resultData.addAllUrl(responseData.getChildUrlSet());

            final RequestData requestData = responseData.getRequestData();
            resultData.removeUrl(requestData);
            resultData.removeUrl(getDuplicateUrl(requestData));