}
	if crawled[url] {
		return
	}
	crawled[url] = true

	wg.Add(1)
	go func() {
		urlq <- url
	}()
}

func addProblem(url, errmsg string) {
	msg := fmt.Sprintf("Error on %s: %s (from %s)", url, errmsg, linkSources[url])
	if *verbose {
		log.Print(msg)
	}
	problems = append(problems, msg)
}

func crawlLoop() {
	for url := range urlq {
		if err := doCrawl(url); err != nil {

       child.scm.url.inherit.append.path="false">
    <connection>scm:my-scm:http://domain.org/base</connection>
    <developerConnection>scm:my-scm:https://domain.org/base/</developerConnection>
    <url>https://domain.org/base</url>
  </scm>
  <distributionManagement>
    <site child.site.url.inherit.append.path="false">
      <url>scp://scp.domain.org/base/</url>
    </site>
  </distributionManagement>

       child.scm.url.inherit.append.path="true">
    <connection>scm:my-scm:http://domain.org/base</connection>
    <developerConnection>scm:my-scm:https://domain.org/base/</developerConnection>
    <url>https://domain.org/base</url>
  </scm>
  <distributionManagement>
    <site child.site.url.inherit.append.path="true">
      <url>scp://scp.domain.org/base/</url>
    </site>
  </distributionManagement>

        final Map<String, Object> requestBody = new HashMap<>();
        final String urls = "https://www.codelibs.org/" + "\n" + "http://failure.url";
        final String includedUrls = "https://www.codelibs.org/.*" + "\n" + "http://failure.url.*";
        requestBody.put("name", NAME_PREFIX + "WebConfig");
        requestBody.put("urls", urls);
        requestBody.put("included_urls", includedUrls);

labels.excludedUrls=Excluded URLs For Crawling
labels.excludedDocPaths=Excluded Paths For Indexing
labels.excludedDocUrls=Excluded URLs For Indexing
labels.hostname=Hostname
labels.id=ID
labels.includedPaths=Included Paths For Crawling
labels.includedUrls=Included URLs For Crawling
labels.includedDocPaths=Included Paths For Indexing
labels.includedDocUrls=Included URLs For Indexing
labels.maxAccessCount=Max Access Count

  }
  return result
}

internal fun canonicalUrl(url: String): String {
  // Silently replace web socket URLs with HTTP URLs.
  return when {
    url.startsWith("ws:", ignoreCase = true) -> {
      "http:${url.substring(3)}"
    }
    url.startsWith("wss:", ignoreCase = true) -> {
      "https:${url.substring(4)}"
    }
    else -> url
  }
}

fun Request.Builder.commonHeader(
  name: String,

Test site for fess-suggest
=================================

## Usage

Start site.
```
$ activator run
```

Finish site. Ctrl + D

### URLs
* test-site : http://localhost:9000

  # Find any "Streaming build results to" lines,
  # de-duplicate,
  # and print the last word from each
  awk '/Streaming build results to/ {print $NF}' "$TFCI_OUTPUT_DIR/script.log" | uniq
}

# Print out any ResultStore URLs for Bazel invocations' results.
# Each failed target there will have its own representation, making failures
# easier to find and read.
function resultstore_extract {
  local PYTHON_BIN XML_PATH

  <artifactId>inheritance</artifactId><!-- same as directory name -->
  <name>Model urls inheritance test child</name>

  <scm>
    <url>https://domain.org/override</url><!-- check that this won't impact child.site.url.inherit.append.path attribute inheritance -->
  </scm>
  <distributionManagement>
    <site><!-- overriding 1 element will reset all elements, but not child.site.url.inherit.append.path attribute -->
      <name>reset</name>
    </site>

```kotlin
cache.evictAll()
```

Removing individual items can be done using the urls iterator.
This would be typical after a user initiates a force refresh by a pull to refresh type action.

```java
    val urlIterator = cache.urls()
    while (urlIterator.hasNext()) {
      if (urlIterator.next().startsWith("https://www.google.com/")) {
        urlIterator.remove()
      }