if (clientConfigMap != null) {
            paramMap.putAll(clientConfigMap);
        }

        // robots txt enabled
        if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
            paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
        }

        final String userAgent = getUserAgent();