while ((c = reader.read()) != -1) {
                    if (Character.isISOControl(c) || isSpaceChar(c)) {
                        if (duplicateTermRemoved) {
                            if (alphanumSize > 0) {
                                isSpace = removeLastDuplication(buf, alphanumSize, isSpace, termCache);
                            } else if (symbolSize > 0) {

         * @throws InvalidKeyException If the access key is invalid
         * @throws ErrorResponseException If the server returns an error
         * @throws IllegalArgumentException If the arguments are invalid
         * @throws InsufficientDataException If insufficient data is available
         * @throws InternalException If an internal error occurs
         * @throws InvalidResponseException If the response is invalid

                line = stripComment(line).trim();
                if (StringUtil.isEmpty(line)) {
                    continue;
                }

                String value = getValue(USER_AGENT_RECORD, line);
                if (value != null) {
                    if (isGroupRecordStarted) {
                        currentDirectiveList.clear();

            final char c = cmd.charAt(i);
            if (c == ' ' && !singleQuote && !doubleQuote && buf.length() > 0) {
                cmdList.add(getCommandValue(buf.toString(), params));
                buf.delete(0, buf.length());
            } else if (c == '\"' && !singleQuote && prevChar != '\\') {
                if (doubleQuote) {
                    if (buf.length() > 0) {

        if (urlQueue != null) {
            if (crawlingQueue.size() > maxCrawlingQueueSize) {
                crawlingQueue.poll();
            }
            crawlingQueue.add(urlQueue);
            return urlQueue;
        }

        synchronized (queueHolder) {
            urlQueue = waitingQueue.poll();
            if (urlQueue == null) {

                if (attrNode != null) {
                    String attrValue = attrNode.getNodeValue();
                    if (StringUtil.isNotBlank(attrValue)) {
                        // if starting with www, append a protocol
                        if (attrValue.startsWith("www.")) {
                            attrValue = "http://" + attrValue;
                        }
                        return attrValue;

     * @throws IOException if an I/O error occurs
     */
    @Override
    public int read(final byte[] b) throws IOException {
        return inputStream.read(b);
    }

    /**
     * Repositions this stream to the position at the time the mark method was last called.
     *

     * If the response body is stored as a file, returns a FileInputStream.
     *
     * @return the response body as an InputStream, or null if no response body is available
     * @throws IORuntimeException if an I/O error occurs while reading the response body file
     */
    public InputStream getResponseBody() {
        if (responseBodyBytes != null) {
            return new ByteArrayInputStream(responseBodyBytes);
        }

        Long millis = params.get("delayMillisAfterProcessing");
        if (millis != null) {
            delayMillisAfterProcessing = millis;
        }

        millis = params.get("delayMillisAtNoUrlInQueue");
        if (millis != null) {
            delayMillisAtNoUrlInQueue = millis;
        }

        millis = params.get("delayMillisBeforeProcessing");
        if (millis != null) {
            delayMillisBeforeProcessing = millis;

        this.logLevel = logLevel;
    }

    /**
     * Checks if the log level is DEBUG.
     *
     * @return true if DEBUG, false otherwise
     */
    public boolean isDebugEnabled() {
        return DEBUG.equals(logLevel);
    }

    /**
     * Checks if the log level is INFO.
     *
     * @return true if INFO, false otherwise
     */
    public boolean isInfoEnabled() {