final SitemapUrl sitemapUrl = new SitemapUrl();
                    sitemapUrl.setLoc(url);
                    sitemapSet.addSitemap(sitemapUrl);
                }
            }
            return sitemapSet;
        } catch (final Exception e) {
            throw new SitemapsException("Could not parse Text Sitemaps.", e);
        }

    }

    public void addDirective(final Directive directive) {
        directiveMap.put(Pattern.compile(directive.getUserAgent().replace("*", ".*"), Pattern.CASE_INSENSITIVE), directive);
    }

    public void addSitemap(final String url) {
        sitemapList.add(url);
    }

    public String[] getSitemaps() {
        return sitemapList.toArray(new String[sitemapList.size()]);
    }

    public static class Directive {

    public static final String INDEX = "Index";

    private final List<Sitemap> sitemapList = new ArrayList<>();

    private String type = URLSET;

    public void addSitemap(final Sitemap sitemap) {
        sitemapList.add(sitemap);
    }

    public void removeSitemap(final Sitemap sitemap) {
        sitemapList.remove(sitemap);
    }

    public Sitemap[] getSitemaps() {

                            }
                        }
                    } else if ((value = getValue(SITEMAP_RECORD, line)) != null && value.length() > 0) {
                        robotsTxt.addSitemap(value);
                    }
                }
            }

            return robotsTxt;
        } catch (final Exception e) {
            throw new RobotsTxtException("Failed to parse robots.txt.", e);

        return maxAccessCount;
    }

    public void setMaxAccessCount(final long maxAccessCount) {
        this.maxAccessCount = maxAccessCount;
    }

    public void addSitemaps(final String[] sitemaps) {
        sitemapsLocal.set(sitemaps);
    }

    public String[] removeSitemaps() {
        final String[] sitemaps = sitemapsLocal.get();
        if (sitemaps != null) {

                    if (robotsTxt != null) {
                        final String[] sitemaps = robotsTxt.getSitemaps();
                        if (sitemaps.length > 0) {
                            crawlerContext.addSitemaps(sitemaps);
                        }

                        final RobotsTxt.Directive directive = robotsTxt.getMatchedDirective(userAgent);
                        if (directive != null) {