if (logger.isDebugEnabled()) {
                        logger.debug("Skipping crawl request for url='{}' at depth={} (maxDepth={}, alreadyProcessed={})",
                                crawlRequest.getUrl(), crawlRequest.getDepth(), maxDepth, processedUrls.contains(crawlRequest.getUrl()));
                    }
                    continue;

                    appendJson("user-info-id", entity.getUserInfoId(), buf).append(',');
                    appendJson("doc-id", entity.getDocId(), buf).append(',');
                    appendJson("url", entity.getUrl(), buf);
                    buf.append('}');
                    buf.append('\n');
                    try {
                        writer.write(buf.toString());
                    } catch (final IOException e) {

    public void test_chatSourceDefaultConstructor() {
        final ChatSource source = new ChatSource();
        assertEquals(0, source.getIndex());
        assertNull(source.getTitle());
        assertNull(source.getUrl());
        assertNull(source.getDocId());
        assertNull(source.getSnippet());
    }

    @Test
    public void test_chatSourceFromMap() {
        final Map<String, Object> doc = new HashMap<>();

        assertEquals("plugin/repo3/fess-crawler-smbj/14.14.0/fess-crawler-smbj-14.14.0.jar", list.get(0).getUrl());
        assertEquals("fess-crawler-smbj", list.get(1).getName());
        assertEquals("14.15.0", list.get(1).getVersion());
        assertEquals("plugin/repo3/fess-crawler-smbj/14.15.0/fess-crawler-smbj-14.15.0.jar", list.get(1).getUrl());
    }

    @Test
    public void test_getArtifactFromFileName1() {

     * @return the URL string or a default value if not extractable
     */
    protected String getUrl(final Object keyObj) {
        if (keyObj instanceof final UrlQueue<?> urlQueue) {
            return escapeValue(urlQueue.getUrl());
        }
        if (keyObj instanceof final StatsKeyObject statsKey) {
            return escapeValue(statsKey.getUrl());
        }
        if (keyObj instanceof final String key) {

                        logger.warn("No data: url={}", accessResult.getUrl());
                        continue;
                    }

                    if (Constants.FALSE.equals(map.get(Constants.INDEXING_TARGET))) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("Skipped indexing (not a target): url={}", accessResult.getUrl());
                        }
                        continue;

    @Override
    public ResultData transform(final ResponseData responseData) {
        if (responseData == null || !responseData.hasResponseBody()) {
            final String url = responseData != null ? responseData.getUrl() : "unknown";
            throw new CrawlingAccessException("No response body for URL: " + url + ". Cannot transform empty response.");
        }

        final ResultData resultData = new ResultData();

        if (childUrlSet != null) {
            // add an url
            try {
                storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
                        urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
            } catch (final Throwable t) {
                if (!ComponentUtil.available()) {

        urlList = fessXpathTransformer.convertChildUrlList(urlList);
        assertEquals(2, urlList.size());
        assertEquals("http://www.example.com", urlList.get(0).getUrl());
        assertEquals("http://www.test.com", urlList.get(1).getUrl());

        urlList.clear();
        urlList.add(RequestDataBuilder.newRequestData().get().url("feed://www.example.com").build());

        final ResponseData responseData = new ResponseData();

        responseData.setUrl("http://example.com/document.pdf");
        assertEquals("http://example.com/document.pdf", responseData.getUrl());

        responseData.setMimeType("application/pdf");
        assertEquals("application/pdf", responseData.getMimeType());

        responseData.setCharSet("UTF-8");