protected function configureRobotsFile(RobotsFileInterface $robots)
    {
        $robots->setCrawlDelay($this->robots['crawl_delay']);
        foreach ($this->robots['allow'] as $path => $userAgent) {
            if (strpos($path, '/') === false) {
                $path = $this->router->generate($path);
            }
            $robots->addAllowEntry($path, $userAgent);
        }
        foreach ($this->robots['disallow'] as $path => $userAgent) {
            if (strpos($path, '/') === false) {
                $path = $this->router->generate($path);
            }
            $robots->addDisallowEntry($path, $userAgent);
        }
        foreach ($this->robots['clean_param'] as $path => $value) {
            $robots->addCleanParamEntry($value['parameters'], $path);
        }
    }
protected function configureRobotsFile(RobotsFileInterface $robots)
    {
        $robots->setCrawlDelay($this->robots['crawl_delay']);
        foreach ($this->robots['allow'] as $path => $userAgent) {
            if (strpos($path, '/') === false) {
                $path = $this->router->generate($path);
            }
            $robots->addAllowEntry($path, $userAgent);
        }
        foreach ($this->robots['disallow'] as $path => $userAgent) {
            if (strpos($path, '/') === false) {
                $path = $this->router->generate($path);
            }
            $robots->addDisallowEntry($path, $userAgent);
        }
        foreach ($this->robots['clean_param'] as $path => $value) {
            $robots->addCleanParamEntry($value['parameters'], $path);
        }
    }
protected function configureRobotsFile ( RobotsFileInterface $ robots ) { $ robots -> setCrawlDelay ( $ this -> robots [ 'crawl_delay' ] ) ; foreach ( $ this -> robots [ 'allow' ] as $ path => $ userAgent ) { if ( strpos ( $ path , '/' ) === false ) { $ path = $ this -> router -> generate ( $ path ) ; } $ robots -> addAllowEntry ( $ path , $ userAgent ) ; } foreach ( $ this -> robots [ 'disallow' ] as $ path => $ userAgent ) { if ( strpos ( $ path , '/' ) === false ) { $ path = $ this -> router -> generate ( $ path ) ; } $ robots -> addDisallowEntry ( $ path , $ userAgent ) ; } foreach ( $ this -> robots [ 'clean_param' ] as $ path => $ value ) { $ robots -> addCleanParamEntry ( $ value [ 'parameters' ] , $ path ) ; } }
Modify robots file by configuration

@param RobotsFileInterface $robots
Modify robots file by configuration