for (int idx = 0; idx < synonyms.size(); idx++) {
            final MyToken synonym = synonyms.get(idx);
            tokenizePartialBlock(nextStart, synonym.startOffset, afterSynonymProduced);

            // enqueue prev-synonym
            processPrevSynonym(synonym.startOffset, idx > 0 ? synonyms.get(idx - 1).endOffset : 0);

            if (expand) {

        if (name == null || enc == null) {
            return null;
        }

        name = name.replaceAll("/+$", "");
        final int idx = name.lastIndexOf('/');
        if (idx >= 0) {
            name = name.substring(idx + 1);
        }
        try {
            return URLDecoder.decode(name, enc);
        } catch (final UnsupportedEncodingException e) {

            return null;
        }

        // trim
        String url = u.trim();

        int idx = url.indexOf('#');
        if (idx >= 0) {
            url = url.substring(0, idx);
        }

        url = url.replace("/./", "/");

        idx = url.indexOf(";jsessionid");
        if (idx >= 0) {
            url = url.replaceFirst(";jsessionid=[a-zA-Z0-9\\.]*", "");
        }

        if (crawlerContext == null) {
            // wrong state
            return;
        }

        final int idx = url.indexOf('/', url.indexOf("://") + 3);
        String hostUrl;
        if (idx >= 0) {
            hostUrl = url.substring(0, idx);
        } else {
            hostUrl = url;
        }
        final String robotTxtUrl = hostUrl + "/robots.txt";

        // check url