if (logger.isDebugEnabled()) {
                logger.debug("{} is already visited.", robotTxtUrl);
            }
            return;
        }

        if (logger.isInfoEnabled()) {
            logger.info("Checking URL: {}", robotTxtUrl);
        }
        // add url to a set
        crawlerContext.getRobotsTxtUrlSet().add(robotTxtUrl);

    public SearchApiManager() {
        setPathPrefix("/api/v1");
    }

    /**
     * Registers this API manager.
     */
    @PostConstruct
    public void register() {
        if (logger.isInfoEnabled()) {
            logger.info("Load {}", this.getClass().getSimpleName());
        }
        ComponentUtil.getWebApiManagerFactory().add(this);
    }

    @Override