if (commentIndex != -1) {
            return line.substring(0, commentIndex);
        }
        return line;
    }

    /**
     * Checks if robots.txt processing is enabled.
     * @return true if enabled, false otherwise
     */
    public boolean isEnabled() {
        return enabled;
    }

    /**
     * Sets whether robots.txt processing is enabled.

                final Boolean b = (Boolean) xObj.value();
                return new String[] { b.toString() };
            case NUMBER:
                final Number d = (Number) xObj.value();
                return new String[] { d.toString() };
            case STRING:
                final String str = (String) xObj.value();
                return new String[] { str.trim() };
            case NODESET:

            return true;
        }
        return false;
    }

    @Override
    public int hashCode() {
        return java.util.Objects.hash(loc, changefreq, lastmod, priority);
    }

    /**
     * Returns a string representation of this SitemapUrl.
     * @return A string representation.
     */
    @Override
    public String toString() {

        if (data == null) {
            return null;
        }
        try {
            return new String(data, StringUtil.isNotBlank(encoding) ? encoding : Constants.UTF_8);
        } catch (final UnsupportedEncodingException e) {
            return new String(data, Constants.UTF_8_CHARSET);
        }
    }

    /**
     * Returns a string representation of this object.

     *
     * @param host the host string to decode
     * @return the decoded host string
     */
    protected String decode(final String host) {
        if (host.indexOf('%') == -1) {
            return host;
        }
        try {
            return URLDecoder.decode(host, encoding);
        } catch (final UnsupportedEncodingException e) {
            return host;
        }
    }

    /**

        if (!filePath.startsWith("storage:")) {
            filePath = "storage://" + filePath;
        }

        return filePath;
    }

    /**
     * Returns the character set used for content encoding.
     * @return the charset
     */
    public String getCharset() {
        return charset;
    }

    /**
     * Sets the character set used for content encoding.

            matchCallCount++;
            return matchResult;
        }

        @Override
        public String getRuleId() {
            return ruleId;
        }

        @Override
        public ResponseProcessor getResponseProcessor() {
            return responseProcessor;
        }

        public int getMatchCallCount() {
            return matchCallCount;
        }
    }

    /**

     * Checks if the log level is DEBUG.
     *
     * @return true if DEBUG, false otherwise
     */
    public boolean isDebugEnabled() {
        return DEBUG.equals(logLevel);
    }

    /**
     * Checks if the log level is INFO.
     *
     * @return true if INFO, false otherwise
     */
    public boolean isInfoEnabled() {
        return INFO.equals(logLevel);
    }

    /**

        this.componentName = componentName;
    }

    /**
     * Returns the onDestroy listener.
     * @return The onDestroy listener.
     */
    public OnDestroyListener<T> getOnDestroyListener() {
        return onDestroyListener;
    }

    /**
     * Sets the onDestroy listener.
     * @param onDestroyListener The OnDestroyListener to set.
     */

        if (urlQueue == null || StringUtil.isBlank(urlQueue.getUrl())
                || crawlerContext.getMaxDepth() >= 0 && urlQueue.getDepth() > crawlerContext.getMaxDepth()) {
            return false;
        }

        // url filter
        if (crawlerContext.urlFilter.match(urlQueue.getUrl())) {
            return true;
        }

        return false;
    }

    /**