symmetry = uint64(len(p.Seq))%ss == 0
				} else {
					symmetry = ss%uint64(len(p.Seq)) == 0
				}
			}
		}
		// With no arg patterns, it is expected that user knows
		// the right symmetry, so either ellipses patterns are
		// provided (recommended) or no ellipses patterns.
		if _, ok := newSetCounts[ss]; !ok && (symmetry || argPatterns == nil) {
			newSetCounts[ss] = struct{}{}
		}
	}

	setCounts = []uint64{}

     * (java.lang.String)
     */
    /**
     * Retrieves a list of URL patterns to be excluded for a given session.
     * @param sessionId The ID of the session.
     * @return A list of compiled regular expression patterns.
     */
    @Override
    public List<Pattern> getExcludeUrlPatternList(final String sessionId) {
        return dataHelper.getExcludeUrlPatternList(sessionId);
    }

     * The user-agent pattern in the directive is converted to a regular expression pattern,
     * where '*' is replaced with '.*' for pattern matching, and stored case-insensitively.
     *
     * @param directive The directive to add to the robots.txt rules
     */
    public void addDirective(final Directive directive) {

    </magic>
    <glob pattern="*.bin"/>
    <glob pattern="*.dms"/>
    <glob pattern="*.lha"/>
    <glob pattern="*.lrf"/>
    <glob pattern="*.lzh"/>
    <glob pattern="*.so"/>
    <glob pattern="*.dist"/>
    <glob pattern="*.distz"/>
    <glob pattern="*.pkg"/>
    <glob pattern="*.bpk"/>
    <glob pattern="*.dump"/>
    <glob pattern="*.elc"/>
    <glob pattern="*.deploy"/>

 * @param historyDays days number of days to store build history .
 * @param artifactsDays number of days to store artifacts. In the stored history, artifacts older than this number will be cleaned up.
 * @param artifactPatterns patterns for artifacts clean-up. If not specified, all artifacts will be removed.
 */
fun Project.cleanupRule(
    historyDays: Int,
    artifactsDays: Int,
    artifactsPatterns: String? = null,
) {
    features {

    /** Idle connection timeout in milliseconds */
    protected long idleConnectionTimeout = 60 * 1000L; // 1min

    /** Pattern for matching HTTP redirect status codes */
    protected Pattern redirectHttpStatusPattern = Pattern.compile("[3][0-9][0-9]");

    /** Whether to use robots.txt disallow rules */
    protected boolean useRobotsTxtDisallows = true;

    /**
     * Pattern for XML tags.
     */
    protected Pattern xmlTagPattern = Pattern.compile("<[^>]+>");

    /**
     * Returns the encoding pattern.
     * @return The encoding pattern.
     */
    @Override
    protected Pattern getEncodingPattern() {
        return xmlEncodingPattern;
    }

    /**

    protected static final Pattern ALLOW_RECORD = Pattern.compile("^allow:\\s*([^\\s]*)\\s*$", Pattern.CASE_INSENSITIVE);

    /** Pattern for parsing crawl-delay records. */
    protected static final Pattern CRAWL_DELAY_RECORD = Pattern.compile("^crawl-delay:\\s*([^\\s]+)\\s*$", Pattern.CASE_INSENSITIVE);

    /**
     * Pattern for Sitemap record.
     */

if [ "$sete" = true ]; then
	set -e
fi

pattern="$1"
if [ "$pattern" = "" ]; then
	pattern=.
fi

./make.bash || exit 1
GOROOT="$(cd .. && pwd)"

gettargets() {
	../bin/go tool dist list | sed -e 's|/|-|' |
		grep -E -v '^(android|ios)' # need C toolchain even for cross-compiling
	echo linux-arm-arm5
}

selectedtargets() {
	gettargets | grep -E "$pattern"
}

crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns
crawler.urlFilter.addInclude("https://example.com/.*");
crawler.urlFilter.addInclude(".*\\.pdf$");

// Exclude patterns  
crawler.urlFilter.addExclude(".*\\.js$");
crawler.urlFilter.addExclude(".*login.*");
```

## Supported Protocols and Formats