* directory.
     */
    protected String path;

    /**
     * A string to replace ?.
     */
    protected String questionStr = "_QUEST_";

    /**
     * A string to replace :.
     */
    protected String colonStr = "_CLN_";

    /**
     * A string to replace ;.
     */
    protected String semicolonStr = "_SCLN_";

    /**
     * A string to replace &.
     */

     */
    protected Map<String, String> convertMap = new LinkedHashMap<>();

    /**
     * Converts a URL.
     *
     * @param url The URL.
     * @return A converted URL.
     */
    public String convert(final String url) {
        if (url == null) {
            return null;
        }
        String convertedUrl = url;
        for (final Map.Entry<String, String> entry : convertMap.entrySet()) {

     * If the encoded string exceeds the maximum allowed length, it truncates the string
     * to the specified maximum length.
     *
     * @param text the input text to be encoded
     * @return the encoded string, truncated if necessary
     */
    public static String createSuggestTextId(final String text) {

    public ExtractData getText(final InputStream in, final Map<String, String> params) {
        final String resourceName = params == null ? null : params.get(ExtractData.RESOURCE_NAME_KEY);

        String extention;
        String filePrefix;
        if (StringUtil.isNotBlank(resourceName)) {
            final String name = getFileName(resourceName);
            final String[] strings = name.split("\\.");

        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {
            final BufferedInputStream bis = new BufferedInputStream(in);
            final String enc = getEncoding(bis);
            final String content = UNESCAPE_HTML4.translate(new String(InputStreamUtil.getBytes(bis), enc));

    public static final String PROXY_PORT_PROPERTY = "proxyPort";

    /** Property name for proxy authentication scheme setting */
    public static final String PROXY_AUTH_SCHEME_PROPERTY = "proxyAuthScheme";

    /** Property name for proxy credentials setting */
    public static final String PROXY_CREDENTIALS_PROPERTY = "proxyCredentials";

    /** Property name for user agent setting */

public class ProfileUtil {
    private static final String SPRING_PROFILES_ACTIVE = "spring.profiles.active";

    public static void setup() {
        final List<String> list = new ArrayList<String>();

        final String values = System.getProperty(SPRING_PROFILES_ACTIVE);
        if (values != null) {
            for (final String value : values.split(",")) {
                list.add(value);
            }

    /**
     * Constructs a new EncodingHelper.
     */
    public EncodingHelper() {
        // Default constructor
    }

    /**
     * Normalizes an encoding string to its preferred form.
     *
     * @param enc the encoding string to normalize
     * @return the normalized encoding or the default encoding if the input is blank
     */

     */
    @Override
    public InetAddress[] resolve(final String host) throws UnknownHostException {
        return InetAddress.getAllByName(toAscii(host));
    }

    /**
     * Decodes the given host string using the specified encoding.
     *
     * @param host the host string to decode
     * @return the decoded host string
     */
    protected String decode(final String host) {
        if (host.indexOf('%') == -1) {

{
  "dynamic_templates": [
    {
      "strings": {
        "mapping": {
          "type": "keyword"
        },
        "match": "*",
        "match_mapping_type": "string"
      }
    }
  ],
  "properties": {
    "settingsType": {
      "type": "keyword"
    },
    "fieldName": {
      "type": "keyword"
    },
    "readingAnalyzer": {
      "type": "keyword"
    },
    "readingTermAnalyzer": {