}
                    if (noindex) {
                        logger.info("HEADER(robots=noindex): {}", responseData.getUrl());
                        storeChildUrls(responseData, resultData);
                        throw new ChildUrlsException(resultData.getChildUrlSet(), "#processXRobotsTag");
                    }