tatus&&304!==c.status||b(c.responseText)},4!==c.readyState&&c.send(null))};if(c.ajax=f,c.queue=d,c.regex={media:/@media[^\{]+\{([^\{\}]*\{[^\}\{]*\})+/gi,keyframes:/@(?:\-(?:o|moz|webkit)\-)?keyframes[^\{]+\{(?:[^\{\}]*\{[^\}\{]*\})+[^\}]*\}/gi,urls:/(url\()['"]?([^\/\)'"][^:\)'"]+)['"]?(\))/g,findStyles:/@media *([^\{]+)\{([\S\s]+?)$/,only:/(only\s+)?([a-zA-Z]+)\s?/,minw:/\([\s]*min\-width\s*:[\s]*([\s]*[0-9\.]+)(px|em)[\s]*\)/,maxw:/\([\s]*max\-width\s*:[\s]*([\s]*[0-9\.]+)(px|em)[\s]*\)/},c.m...

            url.append(parentUrl);
            concatPath(url, pathAdjustment);
            concatPath(url, childPath);

            return url.toString();
        }

        private void concatPath(StringBuilder url, String path) {
            if (!path.isEmpty()) {
                boolean initialUrlEndsWithSlash = url.charAt(url.length() - 1) == '/';

int numOfThread = 10            // Crawler threads
int maxDepth = -1               // Max depth (-1 = unlimited)
long maxAccessCount = 0         // Max URLs (0 = unlimited)
```

### CrawlerThread (`CrawlerThread.java`)

Worker thread for crawling.

**Flow**: Poll URL → Validate → Get client → Delay → Check last-modified → Execute → Process → Extract children → Queue children → Delay

### CrawlerClientFactory

  Installing Maven
  ----------------

  For complete documentation see https://maven.apache.org/download.html#Installation

  Licensing
  ---------

  Please see the file called LICENSE.

  Maven URLS
  ----------

  Home Page:          https://maven.apache.org
  Downloads:          https://maven.apache.org/download.html
  Release Notes:      https://maven.apache.org/docs/history.html

    }

    /**
     * Test processUrl method with various URL patterns
     */
    public void test_processUrl_basic() {
        String sessionId = "test-session-012";
        urlFilter.init(sessionId);

        // Process URL should handle different URL formats
        urlFilter.processUrl("https://example.com/");
        urlFilter.processUrl("http://test.com/path/to/page");

        final StringBuilder url = new StringBuilder();
        url.append(protocol).append("://").append(host);
        if (port > 0) {
            url.append(":").append(port);
        }
        if (path != null && !path.isEmpty()) {
            if (!path.startsWith("/")) {
                url.append("/");
            }
            url.append(path);
        }
        return url.toString();

    /** Indicates processing child URLs due to an exception. */
    PROCESS_CHILD_URLS_BY_EXCEPTION,
    /** Indicates processing a child URL due to an exception. */
    PROCESS_CHILD_URL_BY_EXCEPTION,
    /** Indicates an access exception during crawling. */
    CRAWLING_ACCESS_EXCEPTION,
    /** Indicates a general exception during crawling. */
    CRAWLING_EXCEPTION,
    /** Indicates no URL is available in the queue. */

    }

    /**
     * Gets the URL of the crawled resource.
     *
     * @return the URL
     */
    public String getUrl() {
        return url;
    }

    /**
     * Sets the URL of the crawled resource.
     *
     * @param url the URL to set
     */
    public void setUrl(final String url) {
        this.url = url;
    }

    /**

     */
    @Size(max = 10000)
    public String virtualHostValue;

    /**
     * Enable or disable appending query parameters to search URLs.
     * When enabled, additional parameters are added to search result URLs.
     */
    @Size(max = 10)
    public String appendQueryParameter;

    /**
     * Enable or disable login requirement for search access.

  </description>

  <!-- 5 inherited urls with ${project.directory} added to parent instead of artifactId -->
  <url>http://www.apache.org/child/</url>
  <scm>
    <connection>scm:my-scm:http://domain.org/base/child</connection>
    <developerConnection>scm:my-scm:https://domain.org/base/child/</developerConnection>
    <url>https://domain.org/base/child</url>
  </scm>
  <distributionManagement>
    <site>