if (logger.isDebugEnabled()) {
                        logger.debug("ACE:{}", ace);
                    }
                    processAllowedSIDs(file, ace.getSID(), ace.isAllow() ? sidAllowSet : sidDenySet);
                }
                responseData.addMetaData(SMB_ALLOWED_SID_ENTRIES, sidAllowSet.toArray(new SID[sidAllowSet.size()]));

                    if (logger.isDebugEnabled()) {
                        logger.debug("ACE:{}", ace);
                    }
                    processAllowedSIDs(file, ace.getSID(), ace.isAllow() ? sidAllowSet : sidDenySet);
                }
                responseData.addMetaData(SMB_ALLOWED_SID_ENTRIES, sidAllowSet.toArray(new SID[sidAllowSet.size()]));

            Pattern.compile("^user-agent:\\s*([^\\t\\n\\x0B\\f\\r]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing disallow records. */
    protected static final Pattern DISALLOW_RECORD = Pattern.compile("^disallow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing allow records. */

 *
 * <p>Key features:</p>
 * <ul>
 *   <li>Supports multiple user-agent directives with pattern matching</li>
 *   <li>Handles Allow and Disallow rules for path-based access control</li>
 *   <li>Manages crawl delay settings per user agent</li>
 *   <li>Stores sitemap URLs listed in robots.txt</li>
 * </ul>
 *

        this.redirectHttpStatusPattern = redirectHttpStatusPattern;
    }

    /**
     * Sets whether to use robots.txt disallow rules.
     *
     * @param useRobotsTxtDisallows True to use disallow rules, false otherwise
     */
    public void setUseRobotsTxtDisallows(final boolean useRobotsTxtDisallows) {
        this.useRobotsTxtDisallows = useRobotsTxtDisallows;
    }