private static final String CRAWLER_WEB_PREFIX = "crawler.web.";

    private static final String CRAWLER_WEB_HEADER_PREFIX = CRAWLER_WEB_PREFIX + "header.";

    private static final String CRAWLER_WEB_AUTH = CRAWLER_WEB_PREFIX + "auth";

    private static final String CRAWLER_USERAGENT = "crawler.useragent";

labels.roletype_title_details=Role
labels.roletype_name=Name
labels.roletype_value=Value
labels.reqheader_configuration=Request Header
labels.reqheader_list_name=Name
labels.reqheader_list_web_crawling_config=Config Name
labels.reqheader_create_web_config=Create New Web Config
labels.reqheader_title_details=Request Header
labels.reqheader_name=Name
labels.reqheader_value=Value
labels.reqheader_web_crawling_config=Web Config

            basicAuthList.add(webAuth.getAuthentication());
        }
        paramMap.put(HcHttpClient.AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));

        // request header
        final List<RequestHeader> requestHeaderList = requestHeaderService.getRequestHeaderList(getId());
        final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();

labels.roletype_title_details=Role
labels.roletype_name=Name
labels.roletype_value=Value
labels.reqheader_configuration=Request Header
labels.reqheader_list_name=Name
labels.reqheader_list_web_crawling_config=Config Name
labels.reqheader_create_web_config=Create New Web Config
labels.reqheader_title_details=Request Header
labels.reqheader_name=Name
labels.reqheader_value=Value
labels.reqheader_web_crawling_config=Web Config

                    if (noindex && nofollow) {
                        logger.info("HEADER(robots=noindex,nofollow): {}", responseData.getUrl());
                        throw new ChildUrlsException(Collections.emptySet(), "#processXRobotsTag");
                    }
                    if (noindex) {
                        logger.info("HEADER(robots=noindex): {}", responseData.getUrl());

    /** The path of the HTML: /footer.jsp */
    HtmlNext path_FooterJsp = new HtmlNext("/footer.jsp");

    /** The path of the HTML: /header.jsp */
    HtmlNext path_HeaderJsp = new HtmlNext("/header.jsp");

    /** The path of the HTML: /help.jsp */
    HtmlNext path_HelpJsp = new HtmlNext("/help.jsp");

    /** The path of the HTML: /index.jsp */