*
     * @param sessionId The session ID.
     * @param url The URL.
     * @return The generated unique ID.
     */
    private String getId(final String sessionId, final String url) {
        final String id = sessionId + ID_SEPARATOR + new String(Base64.getUrlEncoder().withoutPadding().encode(url.getBytes(UTF_8)), UTF_8);
        if (id.length() <= idPrefixLength) {
            return id;
        }

     *
     * @return a String representing the metadata.
     */
    String getMetaData();

    /**
     * Sets the metadata for the URL queue.
     *
     * @param metaData the metadata to set
     */
    void setMetaData(String metaData);

    /**
     * Retrieves the encoding of the URL queue.
     *
     * @return the encoding as a String.
     */
    String getEncoding();

    /**

    /** Property name for proxy authentication scheme setting */
    public static final String PROXY_AUTH_SCHEME_PROPERTY = "proxyAuthScheme";

    /** Property name for proxy credentials setting */
    public static final String PROXY_CREDENTIALS_PROPERTY = "proxyCredentials";

    /** Property name for user agent setting */
    public static final String USER_AGENT_PROPERTY = "userAgent";

        runner.refresh();
        settings = Suggester.builder().build(runner.client(), id).settings();
    }

    @Test
    public void test_setAndGetAsArray() {
        String key = "key";
        String value1 = "a";
        String value2 = "b";
        String value3 = "c";
        settings.array().add(key, value1);
        settings.array().add(key, value2);
        settings.array().add(key, value3);

        if (in == null) {
            throw new CrawlerSystemException("The inputstream is null.");
        }
        try {
            final BufferedInputStream bis = new BufferedInputStream(in);
            final String enc = getEncoding(bis);
            final String content = UNESCAPE_HTML4.translate(new String(InputStreamUtil.getBytes(bis), enc));

public class ProfileUtil {
    private static final String SPRING_PROFILES_ACTIVE = "spring.profiles.active";

    public static void setup() {
        final List<String> list = new ArrayList<String>();

        final String values = System.getProperty(SPRING_PROFILES_ACTIVE);
        if (values != null) {
            for (final String value : values.split(",")) {
                list.add(value);
            }

     */
    @Override
    public InetAddress[] resolve(final String host) throws UnknownHostException {
        return InetAddress.getAllByName(toAscii(host));
    }

    /**
     * Decodes the given host string using the specified encoding.
     *
     * @param host the host string to decode
     * @return the decoded host string
     */
    protected String decode(final String host) {
        if (host.indexOf('%') == -1) {

        assertTrue(responseData.getUrl().endsWith("test/text%203.txt"));
        final String content = new String(InputStreamUtil.getBytes(responseData.getResponseBody()), "UTF-8");
        assertEquals("test3\n", content);
    }

    public void test_preprocessUri() {
        String value;
        String result;

        value = "file://test.txt";
        result = "file://test.txt";

    @Override
    protected void setUp() throws Exception {
        super.setUp();

        final Map<String, String> featureMap = newHashMap();
        featureMap.put("http://xml.org/sax/features/namespaces", "false");
        final Map<String, String> propertyMap = newHashMap();
        final Map<String, String> childUrlRuleMap = newHashMap();
        childUrlRuleMap.put("//A", "href");
        childUrlRuleMap.put("//AREA", "href");

     * @throws CrawlingAccessException if an error occurs during message generation
     */
    @Override
    public String generateType3Msg(final String username, final String password, final String domain, final String workstation,
            final String challenge) throws NTLMEngineException {
        Type2Message type2Message;
        try {