User-agent: SapphireWebCrawler
Disallow: /

User-agent: IRLbot
Disallow: /

User-agent: Twiceler
Disallow: / 

User-agent: ShopWiki
Disallow: /

User-agent: Yanga
Disallow: /

User-agent: psbot 
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: CazoodleBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Vagabondo
Disallow: /

User-agent: *
Crawl-Delay: 5

Disallow: /shop/aom/
Disallow: /shop/aom/cache/
Disallow: /shop/links/
Disallow: /shop/text/
Disallow: /uk/shop/aom/
Disallow: /uk/shop/aom/cache/
Disallow: /uk/shop/links/
Disallow: /uk/shop/text/

User-Agent: Googlebot
Disallow: /shop/
Allow: /shop/shop.html
Allow: /shop/UK-*
Allow: /shop/Cookbooks*
Allow: /shop/Recipe*
Allow: /shop/Beef*
Allow: /shop/Chicken*
Allow: /shop/Fish*
Allow: /shop/Lamb*
Allow: /shop/Pork*
Allow: /shop/Bread*
Allow: /shop/Cake*
Allow: /shop/Dessert*
Allow: /shop/Soup*
Allow: /shop/Vegan*
Allow: /shop/Vegetarian*
Allow: /shop/AinsleyHarriot*
Allow: /shop/AntonioCarluccio*
Allow: /shop/AntonyWorrallThompson*
Allow: /shop/ClaudiaRoden*
Allow: /shop/DeliaSmith*
Allow: /shop/ElizabethDavid*
Allow: /shop/GaryRhodes*
Allow: /shop/GordonRamsay*
Allow: /shop/HughFearnleyWhittingstall*
Allow: /shop/KenHom*
Allow: /shop/JamieOliver*
Allow: /shop/KeithFloyd*
Allow: /shop/MadhurJaffrey*
Allow: /shop/MargueritePatten*
Allow: /shop/MaryBerry*
Allow: /shop/MichelRoux*
Allow: /shop/NickNairn*
Allow: /shop/NigelSlater*
Allow: /shop/NigellaLawson*
Allow: /shop/RaymondBlanc*
Allow: /shop/RickStein*
Allow: /shop/RoseElliot*
Allow: /shop/SophieGrigson*
Allow: /shop/GlutenFree*
Allow: /shop/SugarFree*
Allow: /shop/WheatFree*
Allow: /shop/Greek*
Allow: /shop/Italian*
Allow: /shop/Mexican*
Allow: /shop/Breadmaking*
Allow: /shop/CakeDecorating*
Allow: /shop/Breadmakers*
Allow: /shop/MicrowaveOvens*
Allow: /shop/01*
Allow: /shop/02*
Allow: /shop/03*
Allow: /shop/04*
Allow: /shop/05*
Allow: /shop/06*
Allow: /shop/07*
Allow: /shop/08*
Allow: /shop/09*
Allow: /shop/10*
Allow: /shop/11*
Allow: /shop/12*