"type" : "integer"
          },
          "updatedBy" : {
            "type" : "keyword"
          },
          "updatedTime" : {
            "type" : "long"
          },
          "urls" : {
            "type" : "keyword"
          },
          "userAgent" : {
            "type" : "keyword"
          },
          "virtualHosts" : {
            "type" : "keyword"
          }
        }

labels.excludedUrls=URLs a excluir del rastreo
labels.excludedDocPaths=Rutas a excluir de la búsqueda
labels.excludedDocUrls=URLs a excluir de la búsqueda
labels.hostname=Nombre de host
labels.id=ID
labels.includedPaths=Rutas a incluir en el rastreo
labels.includedUrls=URLs a incluir en el rastreo
labels.includedDocPaths=Rutas a incluir en la búsqueda
labels.includedDocUrls=URLs a incluir en la búsqueda

    }

    /**
     * Creates a URL object from the current URL and a relative or absolute URL string.
     *
     * @param currentUrl the current URL as base
     * @param url the URL string to process
     * @return the URL object
     * @throws MalformedURLException if the URL is malformed
     */
    protected URL getURL(final String currentUrl, final String url) throws MalformedURLException {

Isso pode ser útil se, por exemplo, você mora em um país que restringe algumas URLs.

### Desativar a documentação automática

O primeiro passo é desativar a documentação automática, pois por padrão, ela usa o CDN padrão.

Para desativá-los, defina suas URLs como `None` ao criar seu aplicativo `FastAPI`:

{* ../../docs_src/custom_docs_ui/tutorial001.py hl[8] *}

        }
    }

    /**
     * Registers a client component with a regular expression.
     * The component will be loaded into all registered CrawlerClientFactories.
     * @param regex The regular expression to match URLs.
     * @param componentName The name of the component to register.
     */
    public synchronized void register(final String regex, final String componentName) {
        clientMap.put(regex, componentName);

 *
 * <p>
 * It provides methods to set and retrieve the server address, port, username,
 * password, and domain. Additionally, it offers a method to construct a path
 * prefix for SMB URLs based on the configured server and port.
 * </p>
 *
 * <p>
 * The path prefix is in the format "smb://server:port/", where the port is
 * included only if it's greater than 0. If the server is not set, the path

Você pode defini-los da seguinte maneira:

        }

        String url = u;
        final String originalUrl = url;

        int idx = url.indexOf("://");
        if (idx >= 0) {
            url = url.substring(idx + 3);
        }

        idx = url.indexOf('/');
        if (idx >= 0) {
            url = url.substring(0, idx);
        }

        if (url.equals(originalUrl)) {

```java
// Set maximum number of URLs to crawl
crawler.crawlerContext.setMaxAccessCount(1000);

// Set number of crawler threads
crawler.crawlerContext.setNumOfThread(10);

// Set maximum crawl depth
crawler.crawlerContext.setMaxDepth(3);

// Set request interval (politeness)
crawler.crawlerContext.setDefaultIntervalTime(1000); // 1 second
```

### URL Filtering

```java
// Include patterns

      } catch (Exception ignore) {
        // probably URISyntaxException or IllegalArgumentException
        // fall back to copying URLs to files in the testDir == null block below
      }
    }

    if (testDir == null) {
      // testdata resources aren't file:// urls, so create a directory to store them in and then
      // copy the resources to the filesystem as needed
      testDir = createTempDir();
    }