\n"); document.write(" \n"); document.write(" \n"); document.write(" \n"); document.write("\n"); document.write(" \n"); document.write(" \n"); document.write("\n"); document.write("\n"); document.write("\n"); document.write("\n"); document.write("\n"); document.write("\n"); document.write("\n"); document.write(" \n"); document.write("

\n"); document.write(" \n"); document.write(" \n"); document.write("

\n"); document.write(" \n"); document.write("

1\n");
document.write("2\n");
document.write("3\n");
document.write("4\n");
document.write("5\n");
document.write("6\n");
document.write("7\n");
document.write("8\n");
document.write("9\n");
document.write("10\n");
document.write("11\n");
document.write("12\n");
document.write("13\n");
document.write("14\n");
document.write("15\n");
document.write("16\n");
document.write("17\n");
document.write("18\n");
document.write("19\n");
document.write("20\n");
document.write("21\n");
document.write("22\n");
document.write("23\n");
document.write("24\n");
document.write("25\n");
document.write("26\n");
document.write("27\n");
document.write("28\n");
document.write("29\n");
document.write("30\n");
document.write("31\n");
document.write("32\n");
document.write("33\n");
document.write("34\n");
document.write("35\n");
document.write("36\n");
document.write("37\n");
document.write("38\n");
document.write("39\n");
document.write("40\n");
document.write("41\n");
document.write("42\n");
document.write("43\n");
document.write("44\n");
document.write("45\n");
document.write("46\n");
document.write("47\n");
document.write("48\n");
document.write("49\n");
document.write("50\n");
document.write("51\n");
document.write("52\n");
document.write("53\n");
document.write("54\n");
document.write("55\n");
document.write("56\n");
document.write("57\n");
document.write("58\n");
document.write("59\n");
document.write("60\n");
document.write("61\n");
document.write("62\n");
document.write("63\n");
document.write("64\n");
document.write("65\n");
document.write("66\n");
document.write("67\n");
document.write("68\n");
document.write("69\n");
document.write("70\n");
document.write("71\n");
document.write("72\n");
document.write("73\n");
document.write("74\n");
document.write("75\n");
document.write("76\n");
document.write("77\n");
document.write("78\n");
document.write("79\n");
document.write("80\n");
document.write("81\n");
document.write("82\n");
document.write("83\n");
document.write("84\n");
document.write("85\n");
document.write("86\n");
document.write("87\n");
document.write("88\n");
document.write("89\n");
document.write("90\n");
document.write("91\n");
document.write("92\n");
document.write("93\n");
document.write("94\n");
document.write("95\n");
document.write("96\n");
document.write("97\n");
document.write("98\n");
document.write("99\n");
document.write("100\n");
document.write("101\n");
document.write("102\n");
document.write("103\n");
document.write("104\n");
document.write("105\n");
document.write("106\n");
document.write("107\n");
document.write("108\n");
document.write("109\n");
document.write("110\n");
document.write("111\n");
document.write("112\n");
document.write("113\n");
document.write("114\n");
document.write("115\n");
document.write("116\n");
document.write("117\n");
document.write("118\n");
document.write("119\n");
document.write("120\n");
document.write("121\n");
document.write("122\n");
document.write("123\n");
document.write("124\n");
document.write("125\n");
document.write("126\n");
document.write("127\n");
document.write("128\n");
document.write("129\n");
document.write("<\/pre>\n");
document.write("			<\/td>\n");
document.write("

\n"); document.write("

require<\/span> \'anemone\'<\/span><\/span>\n");
document.write("require<\/span> \'nokogiri\'<\/span><\/span>\n");
document.write("\n");
document.write("\n");
document.write("class<\/span> Page < ActiveRecord::Base<\/span><\/span><\/span>\n");
document.write("\n");
document.write("  acts_as_tree :order<\/span> => "title"<\/span>\n");
document.write("\n");
document.write("  def<\/span> set_parent_page_from_last_breadcrumb_item<\/span><\/span>\n");
document.write("    #puts breadcrumb<\/span>\n");
document.write("    bread =<\/span> Nokogiri<\/span>::HTML<\/span>(breadcrumb)\n");
document.write("    the_link =<\/span> nil<\/span>\n");
document.write("    bread.search(\'a\'<\/span>).each do <\/span>|link<\/span>|\n");
document.write("       puts link.content\n");
document.write("       puts link[\'href\'<\/span>]\n");
document.write("       the_link =<\/span> \'http://birmingham.gov.uk\'<\/span> +<\/span> link[\'href\'<\/span>]\n");
document.write("    end<\/span>\n");
document.write("    puts \'Result of link finding: \'<\/span> +<\/span> the_link.to_s\n");
document.write("    unless<\/span> the_link.blank?\n");
document.write("      possible_parent =<\/span> Page<\/span>.find_by_url(the_link)\n");
document.write("      if<\/span> possible_parent\n");
document.write("        puts \'Found parent by url: \'<\/span> +<\/span> possible_parent.title\n");
document.write("\n");
document.write("        self<\/span>.parent_id =<\/span> possible_parent.id\n");
document.write("        self<\/span>.save\n");
document.write("        puts \'Saved new parent id\'<\/span>\n");
document.write("      else<\/span>\n");
document.write("        puts \'No page found by url: \'<\/span> +<\/span> the_link\n");
document.write("\n");
document.write("        new_page =<\/span> Page<\/span>.create_from_anemone_page Anemone<\/span>::Page<\/span>.fetch(the_link)\n");
document.write("        if<\/span> new_page\n");
document.write("          self<\/span>.parent_id =<\/span> new_page.id\n");
document.write("          self<\/span>.save\n");
document.write("        end<\/span>\n");
document.write("\n");
document.write("\n");
document.write("      end<\/span>\n");
document.write("    end<\/span>\n");
document.write("\n");
document.write("  end<\/span>\n");
document.write("\n");
document.write("  def<\/span> self.root_page<\/span><\/span>\n");
document.write("    Page<\/span>.find_by_url(\'http://birmingham.gov.uk/\'<\/span>)\n");
document.write("  end<\/span>\n");
document.write("\n");
document.write("  def<\/span> self.setup_hierarchy<\/span><\/span> # you may need to run this several times manually - sorry.<\/span>\n");
document.write("    pages =<\/span> Page<\/span>.find(:all<\/span>)\n");
document.write("    pages.each do <\/span>|p<\/span>|\n");
document.write("      p.set_parent_page_from_last_breadcrumb_item\n");
document.write("    end<\/span>\n");
document.write("  end<\/span>\n");
document.write("\n");
document.write("  def<\/span> self.create_from_anemone_page<\/span> page<\/span><\/span>\n");
document.write("    puts \'Create from anemone page\'<\/span>\n");
document.write("    #begin<\/span>\n");
document.write("      unless<\/span> page.blank? ||<\/span> page.doc.blank?  \n");
document.write("        html =<\/span> page.doc.at(\'html\'<\/span>)\n");
document.write("        unless<\/span> html.blank?\n");
document.write("          found_page =<\/span> Page<\/span>.find_by_url(page.url.to_s)\n");
document.write("          if<\/span> found_page\n");
document.write("            puts(\'Already indexed. Ignoring: \'<\/span> +<\/span> page.url.to_s)\n");
document.write("          else<\/span>\n");
document.write("\n");
document.write("            title =<\/span> page.doc.at(\'title\'<\/span>)\n");
document.write("            if<\/span> title.blank?\n");
document.write("              puts(\'Blank title. Ignoring: \'<\/span> +<\/span> page.url.to_s)\n");
document.write("            else<\/span>\n");
document.write("\n");
document.write("              if<\/span>(title.inner_html.include?(\'404\'<\/span>) ||<\/span> title.inner_html.downcase.include?(\'error\'<\/span>) )\n");
document.write("                puts(\'404 found. Ignoring: \'<\/span> +<\/span> page.url.to_s)\n");
document.write("              else<\/span>\n");
document.write("                p =<\/span> Page<\/span>.new<\/span>()\n");
document.write("\n");
document.write("\n");
document.write("\n");
document.write("                p.title =<\/span> title.inner_html unless<\/span> title.blank?\n");
document.write("\n");
document.write("                p.url=<\/span> page.url.to_s\n");
document.write("\n");
document.write("                p.page_source=<\/span> html.inner_html \n");
document.write("\n");
document.write("                breadcrumb =<\/span> page.doc.css(\'#breadcrumb\'<\/span>)\n");
document.write("                unless<\/span> breadcrumb.blank?\n");
document.write("                  p.breadcrumb =<\/span> breadcrumb.first.inner_html\n");
document.write("                end<\/span>\n");
document.write("\n");
document.write("                page_content =<\/span> page.doc.css(\'#content\'<\/span>)\n");
document.write("                unless<\/span> page_content.blank?\n");
document.write("                  p.content =<\/span> page_content.first.inner_html\n");
document.write("                end<\/span>\n");
document.write("                if<\/span> p.save\n");
document.write("                  puts(\'Saved: \'<\/span> +<\/span> p.title +<\/span> \' : \'<\/span> +<\/span> p.url.to_s)\n");
document.write("                  return<\/span> p\n");
document.write("                else<\/span>\n");
document.write("                  puts("Save failed: "<\/span> +<\/span> p.url.to_s)\n");
document.write("\n");
document.write("                end<\/span>\n");
document.write("\n");
document.write("              end<\/span> # unless title 404<\/span>\n");
document.write("            end<\/span> # unless title blank<\/span>\n");
document.write("          end<\/span> # unless page already in database<\/span>\n");
document.write("        end<\/span> # unless html blank<\/span>\n");
document.write("      end<\/span> # unless page/doc blank<\/span>\n");
document.write("    #rescue<\/span>\n");
document.write("    #  puts(\'An error occurred. Continuing... \')<\/span>\n");
document.write("    #end #begin/rescue<\/span>\n");
document.write("    return<\/span> false<\/span>\n");
document.write("  end<\/span>\n");
document.write("\n");
document.write("  def<\/span> self.crawl_bcc<\/span> start_url<\/span><\/span>\n");
document.write("    counter =<\/span> 0<\/span>\n");
document.write("    Anemone<\/span>.crawl(start_url) do <\/span>|anemone<\/span>|\n");
document.write("\n");
document.write("      anemone.focus_crawl do <\/span>|page<\/span>| \n");
document.write("        page.links.delete_if { <\/span>|x<\/span>| x.to_s[/<\/span>c=Page&childpagename=SystemAdmin<\/span>/<\/span>] }\n");
document.write("        page.links.delete_if { <\/span>|x<\/span>| x.to_s[/<\/span>pagename=BCC%252FCommon%252FWrapper%252FWrapper<\/span>/<\/span>] }\n");
document.write("        page.links.delete_if { <\/span>|x<\/span>| x.to_s[/<\/span>Common%252FWrapper%252FInlineWrapper<\/span>/<\/span>] }\n");
document.write("        page.links.delete_if { <\/span>|x<\/span>| x.to_s[/<\/span>MungoBlob<\/span>/<\/span>] }\n");
document.write("        #page.links.delete_if { |x| !Page.find_by_url(x.to_s).blank? }<\/span>\n");
document.write("      end<\/span>  \n");
document.write("\n");
document.write("      anemone.on_every_page do <\/span>|page<\/span>|\n");
document.write("        counter =<\/span> counter +<\/span> 1<\/span> if<\/span> Page<\/span>.create_from_anemone_page\n");
document.write("      end<\/span> #on_every_page<\/span>\n");
document.write("    end<\/span> # do Anemone<\/span>\n");
document.write("    puts(counter.to_s +<\/span> \' pages saved\'<\/span>)\n");
document.write("  end<\/span> #self.crawl<\/span>\n");
document.write("\n");
document.write("<\/span>end<\/span> #class<\/span>\n");
document.write("<\/pre><\/pre>\n");
document.write("		<\/td>\n");
document.write("		<\/tr>\n");
document.write("		<\/table>\n");
document.write("	\n");
document.write("	<\/div>\n");
document.write("\n");
document.write("\n");
document.write("  \n");
document.write("  \n");
document.write("  \n");
document.write("    \n");
document.write("      \n");
document.write("      <\/a>\n");
document.write("    <\/div>\n");
document.write("  \n");
document.write("    Pastie #621577<\/a> linked directly from Pastie<\/a>.\n");
document.write("  <\/div><\/div>");