final int pos = text.indexOf(term, offset);
                    if (pos > 0) {
                        final String tmp = text.substring(offset, pos);
                        readingBuf.append(transliterator.transliterate(tmp));
                        offset = pos;
                    } else if (pos == -1) {
                        continue;
                    }

            int offset = 0;
            while (stream.incrementToken()) {
                final CharTermAttribute att = stream.getAttribute(CharTermAttribute.class);
                final String term = att.toString();
                final int pos = inputStr.indexOf(term, offset);
                if (pos > 0) {
                    final String tmp = inputStr.substring(offset, pos);

            String[] words = text.trim().split("[\\s@#$.,!?;:()\\[\\]{}\"']+");

            int position = 0;
            int offset = 0;
            for (String word : words) {
                if (!word.isEmpty()) {
                    int startOffset = text.indexOf(word, offset);
                    int endOffset = startOffset + word.length();

                    // Handle decimal numbers

                buf = new StringBuilder();
            }
        }

        @Override
        public void characters(final char[] ch, final int offset, final int length) {
            if (buf != null) {
                buf.append(new String(ch, offset, length));
            }
        }

        @Override
        public void endElement(final String uri, final String localName, final String qName) {

                buffer.setLength(0);
            }
        }

        @Override
        public void characters(final char[] ch, final int offset, final int length) {
            buffer.append(new String(ch, offset, length));
        }

        @Override
        public void endElement(final String uri, final String localName, final String qName) {
            if ("field".equals(qName)) {

    }

    /**
     * Reads up to len bytes of data from the input stream into an array of bytes.
     *
     * @param b the buffer into which the data is read
     * @param off the start offset in array b at which the data is written
     * @param len the maximum number of bytes to read
     * @return the total number of bytes read into the buffer, or -1 if there is no more data

                        responseData.setResponseBody(outputFile, true);
                    }
                    ftpClientQueue.offer(client);
                } catch (final CrawlingAccessException e) {
                    ftpClientQueue.offer(client);
                    throw e;
                } catch (final Exception e) {
                    logger.warn("I/O Exception.", e);

     * Only URLs that don't already exist will be added.
     *
     * @param sessionId The session ID.
     * @param urlQueueList The list of URL queue entries to offer.
     */
    @Override
    public void offerAll(final String sessionId, final List<OpenSearchUrlQueue> urlQueueList) {
        if (logger.isDebugEnabled()) {