* @param regex The regular expression to match URLs.
     * @param client The CrawlerClient instance.
     */
    public void addClient(final String regex, final CrawlerClient client) {
        if (StringUtil.isBlank(regex)) {
            throw new CrawlerSystemException("A regular expression is null.");
        }
        if (client == null) {
            throw new CrawlerSystemException("CrawlerClient is null.");
        }

     * @param enc the character encoding to use
     * @return the encoded URL
     */
    protected String encodeUrl(final String url, final String enc) {
        if (StringUtil.isBlank(url) || StringUtil.isBlank(enc)) {
            return url;
        }

        final StringBuilder buf = new StringBuilder(url.length() + 100);
        for (final char c : url.toCharArray()) {

fb51f268eeba54%26autoactions%3D1438260718%26uid%3D131239501%26nid%3D244%2B123&amp;t=1&amp;cn=ZmxleGlibGVfcmVjc18y&amp;sig=faadb60a4f6304814f721157367df3076b5a98a2&amp;iid=7c2e8db7f2e0464eb1fb51f268eeba54&amp;uid=131239501&amp;nid=244+123" target="blank" style="text-decoration:none;border-style:none;border:0;padding:0;margin:0;"><img src="https://pbs.twimg.com/profile_images/2997456682/ad8b6c8d2313f963ecbbced0fa1b40f9_reasonably_small.png" class="avatar_med" width="32" height="32" style="margin:0...

            if ("field".equals(qName)) {
                fieldName = attributes.getValue("name");
                if (StringUtil.isBlank(fieldName)) {
                    fieldName = null;
                }
                buffer.setLength(0);
            } else if ("list".equals(qName)) {

     * @param path The FTP path to check.
     * @return true if it matches, false otherwise.
     */
    boolean matches(final String path) {
        if (StringUtil.isBlank(path)) {
            return false;
        }

        try {
            final int pos = path.indexOf('/', 6);
            final URI uri = new URI(pos == -1 ? path : path.substring(0, pos));

         */
        @Override
        public List<AnalyzeToken> analyzeAndReading(final String text, final String field, final String lang) {
            try {
                final String contentsReadingAnalyzerName = getContentsReadingAnalyzerName(field, lang);
                if (StringUtil.isBlank(contentsReadingAnalyzerName)) {
                    return null;

                for (int i = 0; i < tokens.size(); i++) {
                    final AnalyzeToken token = tokens.get(i);
                    final String word = token.getTerm();
                    if (StringUtil.isBlank(word)) {
                        continue;
                    }
                    final String[] words = { word };
                    final String[][] readings = new String[words.length][];

        final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
        if (urlQueue == null) {
            return;
        }

        final String url = urlQueue.getUrl();
        if (StringUtil.isBlank(url) || url.startsWith("file:")) {
            // not target
            return;
        }

        try {
            final URI uri = new URI(url);
            final String host = uri.getHost();

     * @param filename The filename to normalize.
     * @return The normalized filename.
     */
    protected String normalizeFilename(final String filename) {
        if (StringUtil.isBlank(filename)) {
            return filename;
        }
        final StringBuilder buf = new StringBuilder(filename.length() + 10);
        for (int i = 0; i < filename.length(); i++) {

            final int c = bomIn.read(b);

            if (c == -1) {
                return encoding;
            }

            final String head = new String(b, 0, c, encoding);
            if (StringUtil.isBlank(head)) {
                return encoding;
            }
            final Matcher matcher = getEncodingPattern().matcher(head);
            if (matcher.find()) {
                final String enc = matcher.group(1);