for (String urlPattern : directive.getDisallows()) {
                                    if (StringUtil.isNotBlank(urlPattern)) {
                                        urlPattern = convertRobotsTxtPatternToRegex(urlPattern);
                                        final String urlValue = hostUrl + urlPattern;
                                        crawlerContext.getUrlFilter().addExclude(urlValue);