}
                    if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
                        nofollow = true;
                    }
                }
                if (noindex && nofollow) {
                    logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
                    throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
                }