String line;
            final RobotsTxt robotsTxt = new RobotsTxt();
            final List<Directive> currentDirectiveList = new ArrayList<>();
            boolean isGroupRecodeStarted = false;
            while ((line = reader.readLine()) != null) {
                line = stripComment(line).trim();
                if (StringUtil.isEmpty(line)) {
                    continue;
                }

  }

  /**
   * Reads a line of text. A line is considered to be terminated by any one of a line feed ({@code
   * '\n'}), a carriage return ({@code '\r'}), or a carriage return followed immediately by a
   * linefeed ({@code "\r\n"}).
   *
   * @return a {@code String} containing the contents of the line, not including any

 */
class HeadersReader(val source: BufferedSource) {
  private var headerLimit = HEADER_LIMIT.toLong()

  /** Read a single line counted against the header size limit. */
  fun readLine(): String {
    val line = source.readUtf8LineStrict(headerLimit)
    headerLimit -= line.length.toLong()
    return line
  }

  /** Reads headers or trailers. */

                    } else if (line.startsWith(customPromptLiteralStart)) {
                        saveBuffer();
                        const promptStart = line.indexOf(promptLiteralStart);
                        if (promptStart === -1) {
                            console.error("Custom prompt found but no end delimiter", line)
                        }

                }
                valueLines.add(line);
                while (line.length() > 0 && contains(WHITE_SPACE, line.charAt(0))) {
                    line = line.substring(1, line.length());
                }
                buffer.append(line);
                if (!combine) {
                    break;
                }
            }
            return buffer.toString();

		}
	}
	in.Push(NewTokenizer(name, fd, fd))
}

// #line processing.
func (in *Input) line() {
	// Only need to handle Plan 9 format: #line 337 "filename"
	tok := in.Stack.Next()
	if tok != scanner.Int {
		in.expectText("expected line number after #line")
	}
	line, err := strconv.Atoi(in.Stack.Text())
	if err != nil {
		in.Error("error parsing #line (cannot happen):", err)
	}
	tok = in.Stack.Next()

/**
 * Package-protected abstract class that implements the line reading algorithm used by {@link
 * LineReader}. Line separators are per {@link java.io.BufferedReader}: line feed, carriage return,
 * or carriage return followed immediately by a linefeed.
 *
 * <p>Subclasses must implement {@link #handleLine}, call {@link #add} to pass character data, and
 * call {@link #finish} at the end of stream.
 *
 * @author Chris Nokleberg

/**
 * Package-protected abstract class that implements the line reading algorithm used by {@link
 * LineReader}. Line separators are per {@link java.io.BufferedReader}: line feed, carriage return,
 * or carriage return followed immediately by a linefeed.
 *
 * <p>Subclasses must implement {@link #handleLine}, call {@link #add} to pass character data, and
 * call {@link #finish} at the end of stream.
 *
 * @author Chris Nokleberg

    /**
     * Add a header line without any validation. Only appropriate for headers from the remote peer
     * or cache.
     */
    internal fun addLenient(line: String) =
      apply {
        val index = line.indexOf(':', 1)
        when {
          index != -1 -> {
            addLenient(line.substring(0, index), line.substring(index + 1))
          }
          line[0] == ':' -> {

	s.Split(ScanLines)
	for lineNum := 0; s.Scan(); lineNum++ {
		line := lines[lineNum]
		if s.Text() != line {
			t.Errorf("%d: bad line: %d %d\n%.100q\n%.100q\n", lineNum, len(s.Bytes()), len(line), s.Bytes(), line)
		}
	}
	err := s.Err()
	if err != nil {
		t.Fatal(err)
	}
}

// Test that the line splitter handles a final line without a newline.
func TestScanLineNoNewline(t *testing.T) {