labels.web_crawling_configuration=Web Crawling
labels.web_crawling_title_details=Web Crawling Configuration
labels.included_urls=Included URLs For Crawling
labels.excluded_urls=Excluded URLs For Crawling
labels.included_doc_urls=Included URLs For Indexing
labels.excluded_doc_urls=Excluded URLs For Indexing
labels.user_agent=User Agent
labels.web_crawling_button_create=Create
labels.web_crawling_button_create_job=Create new job

labels.web_crawling_configuration=Web Crawling
labels.web_crawling_title_details=Web Crawling Configuration
labels.included_urls=Included URLs For Crawling
labels.excluded_urls=Excluded URLs For Crawling
labels.included_doc_urls=Included URLs For Indexing
labels.excluded_doc_urls=Excluded URLs For Indexing
labels.user_agent=User Agent
labels.web_crawling_button_create=Create
labels.web_crawling_button_create_job=Create new job

            final Map<String, String[]> resultUrlCache = getResultDocIdsCache(session);
            final String[] urls = resultUrlCache.get(queryId);
            if (urls != null) {
                return urls;
            }
            return StringUtil.EMPTY_STRINGS;
        }).orElse(StringUtil.EMPTY_STRINGS);
    }

This could be useful if for example you live in a country that restricts some URLs.

### Disable the automatic docs

The first step is to disable the automatic docs, as by default, those use the default CDN.

To disable them, set their URLs to `None` when creating your `FastAPI` app:

```Python hl_lines="8"
{!../../../docs_src/custom_docs_ui/tutorial001.py!}
```

        if (StringUtil.isNotBlank(webConfigPager.name)) {
            cb.query().setName_Wildcard(wrapQuery(webConfigPager.name));
        }
        if (StringUtil.isNotBlank(webConfigPager.urls)) {
            cb.query().setUrls_Wildcard(wrapQuery(webConfigPager.urls));
        }
        if (StringUtil.isNotBlank(webConfigPager.description)) {
            if (webConfigPager.description.startsWith("*")) {

    assertNotNull(FinalizableReferenceQueue.getStartFinalizer(finalizerCopy));
  }

  static class DecoupledClassLoader extends URLClassLoader {

    public DecoupledClassLoader(URL[] urls) {
      super(urls);
    }

    @Override
    protected synchronized Class<?> loadClass(String name, boolean resolve)
        throws ClassNotFoundException {

  if (result == null) {
    result = CacheControl.parse(headers)
    lazyCacheControl = result
  }
  return result
}

internal fun canonicalUrl(url: String): String {
  // Silently replace web socket URLs with HTTP URLs.
  return when {
    url.startsWith("ws:", ignoreCase = true) -> {
      "http:${url.substring(3)}"
    }
    url.startsWith("wss:", ignoreCase = true) -> {
      "https:${url.substring(4)}"
    }

    assertNotNull(FinalizableReferenceQueue.getStartFinalizer(finalizerCopy));
  }

  static class DecoupledClassLoader extends URLClassLoader {

    public DecoupledClassLoader(URL[] urls) {
      super(urls);
    }

    @Override
    protected synchronized Class<?> loadClass(String name, boolean resolve)
        throws ClassNotFoundException {

            false, false, "Long", 0, 0, null, null, false, null, null, null, null, null, false);
    protected final ColumnInfo _columnUrls = cci("urls", "urls", null, null, String.class, "urls", null, false, false, false, "keyword", 0,
            0, null, null, false, null, null, null, null, null, false);

    private List<Integer> pageNumberList;

    private int pageSize;

    private int currentPageNumber;

    public String id;

    public String name;

    public String urls;

    public String userAgent;

    public String numOfThread;

    public String intervalTime;

    public String boost;

    public String available;

    public String sortOrder;