final String urls = "https://www.codelibs.org/" + "\n" + "http://failure.url";
        final String includedUrls = "https://www.codelibs.org/.*" + "\n" + "http://failure.url.*";
        requestBody.put("name", NAME_PREFIX + "WebConfig");
        requestBody.put("urls", urls);
        requestBody.put("included_urls", includedUrls);
        requestBody.put("user_agent", "Mozilla/5.0");

        }

        if (webConfigList.isEmpty() && fileConfigList.isEmpty()) {
            // nothing
            if (logger.isInfoEnabled()) {
                logger.info("No crawling target urls.");
            }
            return;
        }

        doCrawl(sessionId, webConfigList, fileConfigList);
    }

labels.web_crawling_configuration=Web сканирование
labels.web_crawling_title_details=Конфигурация Web сканирования
labels.included_urls=Включить URLs для сканирования
labels.excluded_urls=Исключить URLs для сканирования
labels.included_doc_urls=Включить URLs для индексации
labels.excluded_doc_urls=Исключить URLs для индексации
labels.user_agent=User Agent
labels.web_crawling_button_create=Создать

labels.web_crawling_configuration=Web Crawling
labels.web_crawling_title_details=Web Crawling Configuration
labels.included_urls=Included URLs For Crawling
labels.excluded_urls=Excluded URLs For Crawling
labels.included_doc_urls=Included URLs For Indexing
labels.excluded_doc_urls=Excluded URLs For Indexing
labels.user_agent=User Agent
labels.web_crawling_button_create=Create
labels.web_crawling_button_create_job=Create new job

labels.web_crawling_configuration=Web Crawling
labels.web_crawling_title_details=Web Crawling Configuration
labels.included_urls=Included URLs For Crawling
labels.excluded_urls=Excluded URLs For Crawling
labels.included_doc_urls=Included URLs For Indexing
labels.excluded_doc_urls=Excluded URLs For Indexing
labels.user_agent=User Agent
labels.web_crawling_button_create=Create
labels.web_crawling_button_create_job=Create new job

            final Map<String, String[]> resultUrlCache = getResultDocIdsCache(session);
            final String[] urls = resultUrlCache.get(queryId);
            if (urls != null) {
                return urls;
            }
            return StringUtil.EMPTY_STRINGS;
        }).orElse(StringUtil.EMPTY_STRINGS);
    }

        if (StringUtil.isNotBlank(webConfigPager.name)) {
            cb.query().setName_Wildcard(wrapQuery(webConfigPager.name));
        }
        if (StringUtil.isNotBlank(webConfigPager.urls)) {
            cb.query().setUrls_Wildcard(wrapQuery(webConfigPager.urls));
        }
        if (StringUtil.isNotBlank(webConfigPager.description)) {
            if (webConfigPager.description.startsWith("*")) {

  if (result == null) {
    result = CacheControl.parse(headers)
    lazyCacheControl = result
  }
  return result
}

internal fun canonicalUrl(url: String): String {
  // Silently replace web socket URLs with HTTP URLs.
  return when {
    url.startsWith("ws:", ignoreCase = true) -> {
      "http:${url.substring(3)}"
    }
    url.startsWith("wss:", ignoreCase = true) -> {
      "https:${url.substring(4)}"
    }

            false, false, "Long", 0, 0, null, null, false, null, null, null, null, null, false);
    protected final ColumnInfo _columnUrls = cci("urls", "urls", null, null, String.class, "urls", null, false, false, false, "keyword", 0,
            0, null, null, false, null, null, null, null, null, false);

    private List<Integer> pageNumberList;

    private int pageSize;

    private int currentPageNumber;

    public String id;

    public String name;

    public String urls;

    public String userAgent;

    public String numOfThread;

    public String intervalTime;

    public String boost;

    public String available;

    public String sortOrder;