assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
    }

    /**
     * @throws Exception
     */
    @Test
    public void testWhitespace() throws Exception {
        final Tokenizer tokenizer = new Tokenizer("\t       \n");
        assertThat(tokenizer.nextToken(), is(Tokenizer.TT_EOF));
    }

    /**

     */
    public String getStringValue() {
        return sval;
    }

    /**
     * Advances to the next token.
     *
     * @return The type of the token.
     */
    public int nextToken() {
        initVal();
        if (processEOF()) {
            return ttype;
        }
        if (processWhitespace()) {
            return ttype;
        }
        if (processWord()) {

                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs.smb1")) {
                        continue;
                    }
                    final String className = provider + "." + protocol + ".Handler";

                final StringTokenizer tokenizer = new StringTokenizer(path, "|");
                while (tokenizer.hasMoreTokens()) {
                    final String provider = tokenizer.nextToken().trim();
                    if (provider.equals("jcifs")) {
                        continue;
                    }
                    final String className = provider + "." + protocol + ".Handler";

            final int len = tok.countTokens();
            final InetAddress[] arr = new InetAddress[len];
            for (int i = 0; i < len; i++) {
                final String addr = tok.nextToken();
                try {
                    arr[i] = InetAddress.getByName(addr);
                } catch (final UnknownHostException uhe) {
                    log.error("Unknown host " + addr, uhe);

     */
    public void setNewToken(final String newToken) {
        this.newToken = newToken;
    }

    /**
     * Returns the new segmentation.
     *
     * @return The new segmentation.
     */
    public String getNewSegmentation() {
        return newSegmentation;
    }

    /**
     * Sets the new segmentation.
     *
     * @param newSegmentation The new segmentation.
     */