{"action":"created","comment":{"url":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/comments/33768703","id":33768703,"diff_hunk":"@@ -0,0 +1,187 @@\n+module SachsenScraper\n+  BASE_URL = 'http://edas.landtag.sachsen.de'\n+  SEARCH_FIELD = 'ctl00$masterContentCallback$content$suchmaske$tblSearch$tabSuche$panelUmSuchmaskeEinfach$suchmaskeEinfachCallback'\n+  SEARCH_FIELD2 =\n+    'ctl00_masterContentCallback_content_suchmaske_tblSearch_tabSuche_panelUmSuchmaskeEinfach_suchmaskeEinfachCallback_sb_Einf'\n+\n+  def self.extract_overview_items(page)\n+    page.search('//td[@class=\"dxdvItem_EDAS\"]/table')\n+  end\n+\n+  def self.extract_title(item)\n+    item.search('.//tr[3]/td/b').text.strip\n+  end\n+\n+  def self.extract_meta_text(item)\n+    text = item.search('.//tr[3]/td').text.split(\"\\n\").find{ |t| t.include?('KlAnfr') || t.include?('GrAnfr') }\n+    text.strip if !text.nil?\n+  end\n+\n+  def self.extractType(text)\n+    if (text[0..5] == 'KlAnfr')\n+      Paper::DOCTYPE_MINOR_INTERPELLATION\n+    else\n+      Paper::DOCTYPE_MAJOR_INTERPELLATION\n+    end\n+  end\n+\n+  def self.extract_date(meta_text)\n+    Date.parse(meta_text.match(/\\d+\\.\\d+\\.\\d{4}/).to_s)\n+  end\n+\n+  def self.extract_reference(meta_text)\n+    meta_text.split('Drs')[1].strip\n+  end\n+\n+  def self.extract_originators(meta_text)\n+    people = [meta_text.partition(/\\p{Upper}{2}/)[0].strip]\n+    parties = [meta_text.match(/\\p{Upper}\\p{Lower}*\\p{Upper}+\\s*\\p{Upper}*/)[0].strip]\n+    originators = { people: people, parties: parties}\n+  end\n+\n+  def self.extract_overview_paper(item, doctype)\n+    meta_text = extract_meta_text item\n+    meta_text = meta_text[6..meta_text.length].strip\n+    full_reference = extract_reference(meta_text)\n+    {\n+      legislative_term: full_reference.split('/')[0],\n+      full_reference: full_reference,\n+      reference: full_reference.split('/')[1],\n+      doctype: doctype,\n+      title: extract_title(item),\n+      url: nil,\n+      published_at: extract_date(meta_text),\n+      originators: extract_originators(meta_text),\n+      is_answer: nil,\n+    }\n+  end\n+\n+  def self.extract_detail_paper(item)\n+    meta_text = extract_meta_text(item)\n+    type = extractType(meta_text)\n+    extract_overview_paper(item, type)\n+  end\n+\n+  def self.switchToTerm(i, m, top)\n+    nav = top.frame_with(name: 'head').click\n+    nav = m.get(\"/redirect.aspx?wahlperiode=\" + i.to_s)\n+    nav.forms.first.submit\n+  end\n+\n+  class Detail < DetailScraper\n+    def scrape\n+      m = mechanize\n+      top = m.get BASE_URL\n+      SachsenScraper.switchToTerm(@legislative_term, m, top)\n+      content = top.frame_with(name: 'content').click\n+      content = searchDetail(content)\n+      paper = nil\n+      SachsenScraper.extract_overview_items(content).each do |item|\n+        paper = SachsenScraper.extract_detail_paper(item)\n+        content = m.click(item.search('//a').first)\n+        answeredDate = content.search('//*[@id=\"ctl00_masterContentCallback_content_tabTreffer_trefferDataView_IT0_HyperLink6\"]').first\n+        if (answeredDate.nil? || answeredDate.text.empty?)\n+          paper[:is_answer] = false\n+        else\n+          paper[:is_answer] = true\n+          paper[:published_at] = SachsenScraper.extract_date(answeredDate.text)\n+          top = m.get(extract_viewer_url(content))\n+          pdfurl = extract_pdf_url(top)\n+          paper[:url] = pdfurl\n+        end\n+      end\n+      paper\n+    end\n+\n+    def extract_pdf_url(top)\n+      nav = top.frame_with(name: 'navigation').click\n+      onloadValue = nav.search(\"//body\").first.attribute(\"onload\")\n+      pdfurl = onloadValue.to_s[/(http\\S*?\\.pdf)/]\n+    end\n+\n+    def extract_viewer_url(content)\n+      pdfTable = content.search('//table[@id=\"ctl00_masterContentCallback_content_tabTreffer_trefferDataView_IT0_anzeige_tblButtons\"]').first\n+      viewerIds = pdfTable.search('//input').select do |i|\n+        !i.attribute('name').nil? && !i.attribute('name').value.nil? && !i.attribute('name').value.index(/no\\$btn/).nil?\n+      end\n+      viewerIds = viewerIds.map do |i|\n+        i.attribute('name').value\n+      end\n+      viewerId = viewerIds.last.match(/anzeige\\$(.*?)_(.*?)_Drs_(.*?)_no\\$btn/)\n+      \"http://edas.landtag.sachsen.de/viewer.aspx?dok_nr=\" + viewerId[1] + \"&dok_art=Drs&leg_per=\" + viewerId[3] + \"&pos_dok=\"+ viewerId[2]\n+    end\n+\n+    def searchDetail(content)\n+      search_form = content.forms.first\n+      search_form[SEARCH_FIELD+'$tf_EinfDoknrVon$ec'] = @reference.to_s","path":"app/scrapers/sachsen_scraper.rb","position":116,"original_position":116,"commit_id":"5610acc3ac478bc97f48f055567ba20a9d898b8c","original_commit_id":"5610acc3ac478bc97f48f055567ba20a9d898b8c","user":{"login":"houndci","id":6697940,"avatar_url":"https://avatars.githubusercontent.com/u/6697940?v=3","gravatar_id":"","url":"https://api.github.com/users/houndci","html_url":"https://github.com/houndci","followers_url":"https://api.github.com/users/houndci/followers","following_url":"https://api.github.com/users/houndci/following{/other_user}","gists_url":"https://api.github.com/users/houndci/gists{/gist_id}","starred_url":"https://api.github.com/users/houndci/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/houndci/subscriptions","organizations_url":"https://api.github.com/users/houndci/orgs","repos_url":"https://api.github.com/users/houndci/repos","events_url":"https://api.github.com/users/houndci/events{/privacy}","received_events_url":"https://api.github.com/users/houndci/received_events","type":"User","site_admin":false},"body":"Surrounding space missing for operator '+'.","created_at":"2015-07-02T11:28:23Z","updated_at":"2015-07-02T11:28:23Z","html_url":"https://github.com/robbi5/kleineanfragen/pull/28#discussion_r33768703","pull_request_url":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/28","_links":{"self":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/comments/33768703"},"html":{"href":"https://github.com/robbi5/kleineanfragen/pull/28#discussion_r33768703"},"pull_request":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/28"}}},"pull_request":{"url":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/28","id":38973191,"html_url":"https://github.com/robbi5/kleineanfragen/pull/28","diff_url":"https://github.com/robbi5/kleineanfragen/pull/28.diff","patch_url":"https://github.com/robbi5/kleineanfragen/pull/28.patch","issue_url":"https://api.github.com/repos/robbi5/kleineanfragen/issues/28","number":28,"state":"open","locked":false,"title":"Sachsen scraper","user":{"login":"globalworming","id":2108984,"avatar_url":"https://avatars.githubusercontent.com/u/2108984?v=3","gravatar_id":"","url":"https://api.github.com/users/globalworming","html_url":"https://github.com/globalworming","followers_url":"https://api.github.com/users/globalworming/followers","following_url":"https://api.github.com/users/globalworming/following{/other_user}","gists_url":"https://api.github.com/users/globalworming/gists{/gist_id}","starred_url":"https://api.github.com/users/globalworming/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/globalworming/subscriptions","organizations_url":"https://api.github.com/users/globalworming/orgs","repos_url":"https://api.github.com/users/globalworming/repos","events_url":"https://api.github.com/users/globalworming/events{/privacy}","received_events_url":"https://api.github.com/users/globalworming/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-07-01T11:36:59Z","updated_at":"2015-07-02T11:28:23Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c5bf406bad249803d6929610aa61d2fe79f9e681","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/28/commits","review_comments_url":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/28/comments","review_comment_url":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/comments{/number}","comments_url":"https://api.github.com/repos/robbi5/kleineanfragen/issues/28/comments","statuses_url":"https://api.github.com/repos/robbi5/kleineanfragen/statuses/5610acc3ac478bc97f48f055567ba20a9d898b8c","head":{"label":"globalworming:sachen-scraper-3","ref":"sachen-scraper-3","sha":"5610acc3ac478bc97f48f055567ba20a9d898b8c","user":{"login":"globalworming","id":2108984,"avatar_url":"https://avatars.githubusercontent.com/u/2108984?v=3","gravatar_id":"","url":"https://api.github.com/users/globalworming","html_url":"https://github.com/globalworming","followers_url":"https://api.github.com/users/globalworming/followers","following_url":"https://api.github.com/users/globalworming/following{/other_user}","gists_url":"https://api.github.com/users/globalworming/gists{/gist_id}","starred_url":"https://api.github.com/users/globalworming/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/globalworming/subscriptions","organizations_url":"https://api.github.com/users/globalworming/orgs","repos_url":"https://api.github.com/users/globalworming/repos","events_url":"https://api.github.com/users/globalworming/events{/privacy}","received_events_url":"https://api.github.com/users/globalworming/received_events","type":"User","site_admin":false},"repo":{"id":30397914,"name":"kleineanfragen","full_name":"globalworming/kleineanfragen","owner":{"login":"globalworming","id":2108984,"avatar_url":"https://avatars.githubusercontent.com/u/2108984?v=3","gravatar_id":"","url":"https://api.github.com/users/globalworming","html_url":"https://github.com/globalworming","followers_url":"https://api.github.com/users/globalworming/followers","following_url":"https://api.github.com/users/globalworming/following{/other_user}","gists_url":"https://api.github.com/users/globalworming/gists{/gist_id}","starred_url":"https://api.github.com/users/globalworming/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/globalworming/subscriptions","organizations_url":"https://api.github.com/users/globalworming/orgs","repos_url":"https://api.github.com/users/globalworming/repos","events_url":"https://api.github.com/users/globalworming/events{/privacy}","received_events_url":"https://api.github.com/users/globalworming/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/globalworming/kleineanfragen","description":"","fork":true,"url":"https://api.github.com/repos/globalworming/kleineanfragen","forks_url":"https://api.github.com/repos/globalworming/kleineanfragen/forks","keys_url":"https://api.github.com/repos/globalworming/kleineanfragen/keys{/key_id}","collaborators_url":"https://api.github.com/repos/globalworming/kleineanfragen/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/globalworming/kleineanfragen/teams","hooks_url":"https://api.github.com/repos/globalworming/kleineanfragen/hooks","issue_events_url":"https://api.github.com/repos/globalworming/kleineanfragen/issues/events{/number}","events_url":"https://api.github.com/repos/globalworming/kleineanfragen/events","assignees_url":"https://api.github.com/repos/globalworming/kleineanfragen/assignees{/user}","branches_url":"https://api.github.com/repos/globalworming/kleineanfragen/branches{/branch}","tags_url":"https://api.github.com/repos/globalworming/kleineanfragen/tags","blobs_url":"https://api.github.com/repos/globalworming/kleineanfragen/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/globalworming/kleineanfragen/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/globalworming/kleineanfragen/git/refs{/sha}","trees_url":"https://api.github.com/repos/globalworming/kleineanfragen/git/trees{/sha}","statuses_url":"https://api.github.com/repos/globalworming/kleineanfragen/statuses/{sha}","languages_url":"https://api.github.com/repos/globalworming/kleineanfragen/languages","stargazers_url":"https://api.github.com/repos/globalworming/kleineanfragen/stargazers","contributors_url":"https://api.github.com/repos/globalworming/kleineanfragen/contributors","subscribers_url":"https://api.github.com/repos/globalworming/kleineanfragen/subscribers","subscription_url":"https://api.github.com/repos/globalworming/kleineanfragen/subscription","commits_url":"https://api.github.com/repos/globalworming/kleineanfragen/commits{/sha}","git_commits_url":"https://api.github.com/repos/globalworming/kleineanfragen/git/commits{/sha}","comments_url":"https://api.github.com/repos/globalworming/kleineanfragen/comments{/number}","issue_comment_url":"https://api.github.com/repos/globalworming/kleineanfragen/issues/comments{/number}","contents_url":"https://api.github.com/repos/globalworming/kleineanfragen/contents/{+path}","compare_url":"https://api.github.com/repos/globalworming/kleineanfragen/compare/{base}...{head}","merges_url":"https://api.github.com/repos/globalworming/kleineanfragen/merges","archive_url":"https://api.github.com/repos/globalworming/kleineanfragen/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/globalworming/kleineanfragen/downloads","issues_url":"https://api.github.com/repos/globalworming/kleineanfragen/issues{/number}","pulls_url":"https://api.github.com/repos/globalworming/kleineanfragen/pulls{/number}","milestones_url":"https://api.github.com/repos/globalworming/kleineanfragen/milestones{/number}","notifications_url":"https://api.github.com/repos/globalworming/kleineanfragen/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/globalworming/kleineanfragen/labels{/name}","releases_url":"https://api.github.com/repos/globalworming/kleineanfragen/releases{/id}","created_at":"2015-02-06T06:06:10Z","updated_at":"2015-05-07T07:49:18Z","pushed_at":"2015-07-02T11:28:17Z","git_url":"git://github.com/globalworming/kleineanfragen.git","ssh_url":"git@github.com:globalworming/kleineanfragen.git","clone_url":"https://github.com/globalworming/kleineanfragen.git","svn_url":"https://github.com/globalworming/kleineanfragen","homepage":"https://kleineanfragen.de","size":5221,"stargazers_count":0,"watchers_count":0,"language":"Ruby","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"robbi5:master","ref":"master","sha":"fe8dacc6739f90d498dc67d087e5d540d13d2d17","user":{"login":"robbi5","id":172415,"avatar_url":"https://avatars.githubusercontent.com/u/172415?v=3","gravatar_id":"","url":"https://api.github.com/users/robbi5","html_url":"https://github.com/robbi5","followers_url":"https://api.github.com/users/robbi5/followers","following_url":"https://api.github.com/users/robbi5/following{/other_user}","gists_url":"https://api.github.com/users/robbi5/gists{/gist_id}","starred_url":"https://api.github.com/users/robbi5/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robbi5/subscriptions","organizations_url":"https://api.github.com/users/robbi5/orgs","repos_url":"https://api.github.com/users/robbi5/repos","events_url":"https://api.github.com/users/robbi5/events{/privacy}","received_events_url":"https://api.github.com/users/robbi5/received_events","type":"User","site_admin":false},"repo":{"id":23664133,"name":"kleineanfragen","full_name":"robbi5/kleineanfragen","owner":{"login":"robbi5","id":172415,"avatar_url":"https://avatars.githubusercontent.com/u/172415?v=3","gravatar_id":"","url":"https://api.github.com/users/robbi5","html_url":"https://github.com/robbi5","followers_url":"https://api.github.com/users/robbi5/followers","following_url":"https://api.github.com/users/robbi5/following{/other_user}","gists_url":"https://api.github.com/users/robbi5/gists{/gist_id}","starred_url":"https://api.github.com/users/robbi5/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robbi5/subscriptions","organizations_url":"https://api.github.com/users/robbi5/orgs","repos_url":"https://api.github.com/users/robbi5/repos","events_url":"https://api.github.com/users/robbi5/events{/privacy}","received_events_url":"https://api.github.com/users/robbi5/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/robbi5/kleineanfragen","description":"Collecting kleine Anfragen from Parlamentsdokumentationssystemen for easy search- and linkability","fork":false,"url":"https://api.github.com/repos/robbi5/kleineanfragen","forks_url":"https://api.github.com/repos/robbi5/kleineanfragen/forks","keys_url":"https://api.github.com/repos/robbi5/kleineanfragen/keys{/key_id}","collaborators_url":"https://api.github.com/repos/robbi5/kleineanfragen/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/robbi5/kleineanfragen/teams","hooks_url":"https://api.github.com/repos/robbi5/kleineanfragen/hooks","issue_events_url":"https://api.github.com/repos/robbi5/kleineanfragen/issues/events{/number}","events_url":"https://api.github.com/repos/robbi5/kleineanfragen/events","assignees_url":"https://api.github.com/repos/robbi5/kleineanfragen/assignees{/user}","branches_url":"https://api.github.com/repos/robbi5/kleineanfragen/branches{/branch}","tags_url":"https://api.github.com/repos/robbi5/kleineanfragen/tags","blobs_url":"https://api.github.com/repos/robbi5/kleineanfragen/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/robbi5/kleineanfragen/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/robbi5/kleineanfragen/git/refs{/sha}","trees_url":"https://api.github.com/repos/robbi5/kleineanfragen/git/trees{/sha}","statuses_url":"https://api.github.com/repos/robbi5/kleineanfragen/statuses/{sha}","languages_url":"https://api.github.com/repos/robbi5/kleineanfragen/languages","stargazers_url":"https://api.github.com/repos/robbi5/kleineanfragen/stargazers","contributors_url":"https://api.github.com/repos/robbi5/kleineanfragen/contributors","subscribers_url":"https://api.github.com/repos/robbi5/kleineanfragen/subscribers","subscription_url":"https://api.github.com/repos/robbi5/kleineanfragen/subscription","commits_url":"https://api.github.com/repos/robbi5/kleineanfragen/commits{/sha}","git_commits_url":"https://api.github.com/repos/robbi5/kleineanfragen/git/commits{/sha}","comments_url":"https://api.github.com/repos/robbi5/kleineanfragen/comments{/number}","issue_comment_url":"https://api.github.com/repos/robbi5/kleineanfragen/issues/comments{/number}","contents_url":"https://api.github.com/repos/robbi5/kleineanfragen/contents/{+path}","compare_url":"https://api.github.com/repos/robbi5/kleineanfragen/compare/{base}...{head}","merges_url":"https://api.github.com/repos/robbi5/kleineanfragen/merges","archive_url":"https://api.github.com/repos/robbi5/kleineanfragen/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/robbi5/kleineanfragen/downloads","issues_url":"https://api.github.com/repos/robbi5/kleineanfragen/issues{/number}","pulls_url":"https://api.github.com/repos/robbi5/kleineanfragen/pulls{/number}","milestones_url":"https://api.github.com/repos/robbi5/kleineanfragen/milestones{/number}","notifications_url":"https://api.github.com/repos/robbi5/kleineanfragen/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/robbi5/kleineanfragen/labels{/name}","releases_url":"https://api.github.com/repos/robbi5/kleineanfragen/releases{/id}","created_at":"2014-09-04T13:47:44Z","updated_at":"2015-05-08T13:07:39Z","pushed_at":"2015-07-02T11:28:18Z","git_url":"git://github.com/robbi5/kleineanfragen.git","ssh_url":"git@github.com:robbi5/kleineanfragen.git","clone_url":"https://github.com/robbi5/kleineanfragen.git","svn_url":"https://github.com/robbi5/kleineanfragen","homepage":"https://kleineanfragen.de","size":5895,"stargazers_count":3,"watchers_count":3,"language":"Ruby","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":2,"forks":2,"open_issues":2,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/28"},"html":{"href":"https://github.com/robbi5/kleineanfragen/pull/28"},"issue":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/issues/28"},"comments":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/issues/28/comments"},"review_comments":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/28/comments"},"review_comment":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/pulls/28/commits"},"statuses":{"href":"https://api.github.com/repos/robbi5/kleineanfragen/statuses/5610acc3ac478bc97f48f055567ba20a9d898b8c"}}}}