# Allow all User-agent: * Disallow: # disallow all files in these directories Disallow: /administration/ Disallow: /themes/ Disallow: /links/ Disallow: /navbar/ Disallow: /scripts/ Disallow: /data/ # disallow all files ending with these extensions Disallow: /*.js$ Disallow: /*.css$ Disallow: /*.txt$ Disallow: /*.gif$ Disallow: /*.png$ Disallow: /*.jpeg$ Disallow: /*.jpg$ Disallow: /*.ico$ Disallow: /*.jpg$ # disallow archiving site User-agent: ia_archiver Disallow: /