}

        super.init();

        // robots.txt parser
        final Boolean robotsTxtEnabled = getInitParameter(ROBOTS_TXT_ENABLED_PROPERTY, Boolean.TRUE, Boolean.class);
        if (robotsTxtHelper != null) {
            robotsTxtHelper.setEnabled(robotsTxtEnabled);
        }

        // httpclient
        final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();