entity.getQueryId(), buf).append(','); L549: appendJson("user-info-id", entity.getUserInfoId(), buf).append(','); L550: appendJson("doc-id", entity.getDocId(), buf).append(','); L551: appendJson("url", entity.getUrl(), buf); L552: buf.append('}'); L553: buf.append('\n'); L554: try { L555: writer.write(buf.toString()); L556: } catch (final IOException e) { L557:...

(cae.isDebugEnabled()) { L129: logger.debug("[{}] Crawling Access Exception at {}", failureUrlId, urlQueue.getUrl(), cae); L130: } else if (cae.isInfoEnabled()) { L131: logger.info("[{}] {}", failureUrlId, cae.getMessage()); L132: } else if (cae.isWarnEnabled()) { L133: logger.warn("[{}] Crawling Access Exception at {}", failureUrlId, urlQueue.getUrl(), cae); L134: } else if (cae.isErrorEnabled()) { L135: logger.error("[{}] Crawling Access Exception...

(canonicalUrl != null && !canonicalUrl.equalsIgnoreCase(responseData.getUrl()) && isValidUrl(canonicalUrl) L350: && isValidCanonicalUrl(responseData.getUrl(), canonicalUrl)) { L351: final Set<RequestData> childUrlSet = new HashSet<>(); L352: childUrlSet.add(RequestDataBuilder.newRequestData().get().url(canonicalUrl).build()); L353: logger.info("CANONICAL: {} -> {}", responseData.getUrl(), canonicalUrl); L354: throw new ChildUrlsException(childUrlSet,...

()); L605: urlList = fessXpathTransformer.convertChildUrlList(urlList); L606: assertEquals(2, urlList.size()); L607: assertEquals("http://www.example.com", urlList.get(0).getUrl()); L608: assertEquals("http://www.test.com", urlList.get(1).getUrl()); L609: L610: urlList.clear(); L611: urlList.add(RequestDataBuilder.newRequestData().get().url("feed://www.example.com").build()); L612: urlList.add(RequestDataBuilder.newRequestData().get().url("http://...

ze = Long.parseLong(fessConfig.getIndexerWebfsMaxDocumentRequestSize()); L327: for (final EsAccessResult accessResult : arList) { L328: if (logger.isDebugEnabled()) { L329: logger.debug("Indexing {}", accessResult.getUrl()); L330: } L331: accessResult.setStatus(Constants.DONE_STATUS); L332: accessResultList.add(accessResult); L333: L334: if (accessResult.getHttpStatusCode() != 200) { L335: // invalid page...

} L160: } L161: } L162: L163: }); L164: } catch (final Exception e) { L165: final CrawlingAccessException rcae = new CrawlingAccessException("Could not get a text from " + responseData.getUrl(), e); L166: rcae.setLogLevel(CrawlingAccessException.WARN); L167: throw rcae; L168: } L169: if (content == null) { L170: content = StringUtil.EMPTY; L171: } L172: final String contentMeta...

install(artifact); L284: break; L285: } L286: } L287: L288: protected void install(final Artifact artifact) { L289: final String fileName = artifact.getFileName(); L290: final String url = artifact.getUrl(); L291: if (StringUtil.isBlank(url)) { L292: throw new PluginException("url is blank: " + artifact.getName()); L293: } L294: if (url.startsWith("http:") || url.startsWith("https:")) { L295: try (final CurlResponse...

ResponseData responseData) { L728: String charset = responseData.getCharSet(); L729: if (charset == null) { L730: charset = Constants.UTF_8; L731: } L732: final String name; L733: final String url = responseData.getUrl(); L734: final int pos = url.lastIndexOf('/'); L735: try { L736: if (pos >= 0 && pos + 1 < url.length()) { L737: name = URLDecoder.decode(url.substring(pos + 1), charset); L738: } else { L739: ...