<components namespace="fessCrawler">
	<include path="crawler/container.xml" />

	<component name="encodingHelper" class="org.codelibs.fess.crawler.helper.EncodingHelper">
		<postConstruct name="addEncodingMapping">
			<arg>"unicode"</arg>
			<arg>"UTF-16LE"</arg>
		</postConstruct>
	</component>

import org.apache.http.conn.DnsResolver;

/**
 * The {@code IdnDnsResolver} class implements the {@code DnsResolver} interface
 * to provide DNS resolution with support for Internationalized Domain Names (IDN).
 * It converts Unicode domain names to ASCII Compatible Encoding (ACE) using the
 * {@link java.net.IDN} class.
 *
 * <p>This class allows setting a custom flag for the IDN conversion and a custom
 * encoding for URL decoding.</p>
 *

        // Test with special characters
        testTransformer.setName("name-with_special.chars#123");
        assertEquals("name-with_special.chars#123", testTransformer.getName());

        // Test with Unicode characters
        testTransformer.setName("変換器の名前");
        assertEquals("変換器の名前", testTransformer.getName());

        // Test with mixed characters
        testTransformer.setName("Transformer-変換器-123");

## Technology Stack

- **Java**: 21+ (configured via parent POM)
- **OpenSearch**: Latest (provided scope)
- **Apache Lucene**: Query parsing and text analysis
- **ICU4J**: Unicode text processing and normalization
- **JUnit 4**: Testing framework
- **Maven**: Build and dependency management

## Quick Start

### Prerequisites

- Java 21 or higher

        // Special characters in rule ID
        testRule.setRuleId("rule-id_with.special@chars#123");
        assertEquals("rule-id_with.special@chars#123", testRule.getRuleId());

        // Unicode characters
        testRule.setRuleId("ルール日本語");
        assertEquals("ルール日本語", testRule.getRuleId());

        // Whitespace
        testRule.setRuleId("  rule with spaces  ");

                    responseParams.stream().forEach(p -> {
                        try {
                            buf.append(URLEncoder.encode(p.getFirst(), encoding));
                            buf.append('=');
                            buf.append(URLEncoder.encode(p.getSecond(), encoding));
                        } catch (final UnsupportedEncodingException e) {
                            throw new IORuntimeException(e);

                    if (StringUtil.isAsciiPrintable(str)) {
                        buf.append(c);
                    } else {
                        buf.append(URLEncoder.encode(str, charset));
                    }
                }
            }
        } catch (final UnsupportedEncodingException e) {
            return filePath;
        }
        return buf.toString();
    }

            logger.warn("Malformed URL: " + attrValue, e);
        }
    }

    /**
     * Encodes a URL using the specified character encoding.
     *
     * @param url the URL to encode
     * @param enc the character encoding to use
     * @return the encoded URL
     */
    protected String encodeUrl(final String url, final String enc) {

     * @return The generated unique ID.
     */
    private String getId(final String sessionId, final String url) {
        final String id = sessionId + ID_SEPARATOR + new String(Base64.getUrlEncoder().withoutPadding().encode(url.getBytes(UTF_8)), UTF_8);
        if (id.length() <= idPrefixLength) {
            return id;
        }
        return id.substring(0, idPrefixLength) + MessageDigestUtil.digest("SHA-256", id.substring(idPrefixLength));

    <sub-class-of type="video/x-ms-asf" />
    <glob pattern="*.wmv"/>
    <magic priority="60">
       <match value="Windows Media Video" type="unicodeLE" offset="0:8192" />
       <match value="VC-1 Advanced Profile" type="unicodeLE" offset="0:8192" />
       <match value="wmv2" type="unicodeLE" offset="0:8192" />
    </magic>
  </mime-type>
  <mime-type type="video/x-ms-wmx">
    <glob pattern="*.wmx"/>
  </mime-type>