public void testParse_disable() {
        final InputStream in = RobotsTxtHelperTest.class.getResourceAsStream("robots.txt");
        robotsTxtHelper.setEnabled(false);
        try {
            assertNull(robotsTxtHelper.parse(in));
        } finally {
            robotsTxtHelper.setEnabled(true);
            CloseableUtil.closeQuietly(in);
        }
    }

    public boolean isEnabled() {
        return enabled;
    }

    /**
     * Sets whether robots.txt processing is enabled.
     * @param enabled true to enable, false to disable
     */
    public void setEnabled(final boolean enabled) {
        this.enabled = enabled;
    }