linebreaks bool // print linebreaks instead of semis

	indent  int // current indentation level
	nlcount int // number of consecutive newlines

	pending []whitespace // pending whitespace
	lastTok token        // last token (after any pending semi) processed by print
}

// write is a thin wrapper around p.output.Write
// that takes care of accounting and error handling.
func (p *printer) write(data []byte) {

            // if we haven't added this arg to the buffer, and we ARE building an argument
            // buffer, then append it with a preceding space...again, not sure what else to
            // do other than collapse whitespace.
            // NOTE: The case of a trailing quote is handled by nullifying the arg buffer.
            if (!addedToBuffer) {
                if (currentArg != null) {
                    currentArg.append(' ').append(arg);

  // In the case of input tensor with 0 rank.
  // Whitespace tokenizer generates 1 output:
  // * String tensor for tokens.
  //
  // In the case of 1-D input tensor,
  // Whitespace tokenizer generates 2 outputs to make up a ragged tensor:
  // * 1st output is the value of ragged tensor;
  // * 2nd output is the offset.
  //
  // In the case of batched input tesnor,
  // Whitespace tokenizer has 3 outputs to make up a nested ragged tensor:

		{[]byte(`-lextra\ fun\ arg\\`), []string{`-lextra fun arg\`}},
		{[]byte("\textra     whitespace\r\n"), []string{"extra", "whitespace\r"}},
		{[]byte("     \r\n      "), []string{"\r"}},
		{[]byte(`"-r:foo" "-L/usr/white space/lib" "-lfoo bar" "-lbar baz"`), []string{"-r:foo", "-L/usr/white space/lib", "-lfoo bar", "-lbar baz"}},

}

// getLine results the first \r\n or \n delineated line from the given byte
// array. The line does not include trailing whitespace or the trailing new
// line bytes. The remainder of the byte array (also not including the new line
// bytes) is also returned and this will always be smaller than the original
// argument.

		{in: "foo bar\xff", want: "foo bar\xff"},

		// Equal sign.
		{in: "=3D30\n", want: "=30\n"},
		{in: "=00=FF0=\n", want: "\x00\xff0"},

		// Trailing whitespace
		{in: "foo  \n", want: "foo\n"},
		{in: "foo  \n\nfoo =\n\nfoo=20\n\n", want: "foo\n\nfoo \nfoo \n\n"},

		// Tests that we allow bare \n and \r through, despite it being strictly

// week.
//
// # Counter Naming
//
// Counter names passed to [New] and [NewStack] should follow these
// conventions:
//
//   - Names cannot contain whitespace or newlines.
//
//   - Names must be valid unicode, with no unprintable characters.
//
//   - Names may contain at most one ':'. In the counter "foo:bar", we refer to

}

func NewTokenizer(name string, r io.Reader, file *os.File) *Tokenizer {
	var s scanner.Scanner
	s.Init(r)
	// Newline is like a semicolon; other space characters are fine.
	s.Whitespace = 1<<'\t' | 1<<'\r' | 1<<' '
	// Don't skip comments: we need to count newlines.
	s.Mode = scanner.ScanChars |
		scanner.ScanFloats |
		scanner.ScanIdents |
		scanner.ScanInts |
		scanner.ScanStrings |

     */
    private void parseDefineDirectiveBody(Buffer buffer, Collection<Macro> macros, Collection<MacroFunction> macroFunctions) {
        if (!buffer.consumeWhitespace()) {
            // No separating whitespace between the #define and the name
            return;
        }
        String name = buffer.readIdentifier();
        if (name == null) {
            // No macro name
            return;
        }

    return CodeLine(absl::Substitute(text, args...));
  }

  // Append a multiline string of source code, left-justified (not indented).
  // Note: Trims leading/trailing whitespace including newlines, making this
  //       method convenient for multiline raw strings.
  // Newlines ('\n') are allowed/expected.
  Renderer &CodeLines(const string &text);
  template <typename... Args>