{"action":"created","comment":{"url":"https://api.github.com/repos/CartoDB/cartodb/pulls/comments/45445150","id":45445150,"diff_hunk":"@@ -125,13 +125,25 @@ def self.supported?(extension)\n       end\n \n       def normalize(temporary_filepath)\n-        temporary_csv = ::CSV.open(temporary_filepath, 'w', col_sep: OUTPUT_DELIMITER)\n-        File.open(filepath, 'rb', external_encoding: encoding)\n-        .each_line(line_delimiter) { |line|\n-          row = parsed_line(line)\n-          next unless row\n-          temporary_csv << multiple_column(row)\n-        }\n+\n+        temporary_csv = CSV.open(temporary_filepath, 'w', col_sep: OUTPUT_DELIMITER, encoding: 'UTF-8')","path":"services/importer/lib/importer/csv_normalizer.rb","position":12,"original_position":12,"commit_id":"3ad0ab575d43402b8311775216fe1f39c23724da","original_commit_id":"3ad0ab575d43402b8311775216fe1f39c23724da","user":{"login":"ethervoid","id":741240,"avatar_url":"https://avatars.githubusercontent.com/u/741240?v=3","gravatar_id":"","url":"https://api.github.com/users/ethervoid","html_url":"https://github.com/ethervoid","followers_url":"https://api.github.com/users/ethervoid/followers","following_url":"https://api.github.com/users/ethervoid/following{/other_user}","gists_url":"https://api.github.com/users/ethervoid/gists{/gist_id}","starred_url":"https://api.github.com/users/ethervoid/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ethervoid/subscriptions","organizations_url":"https://api.github.com/users/ethervoid/orgs","repos_url":"https://api.github.com/users/ethervoid/repos","events_url":"https://api.github.com/users/ethervoid/events{/privacy}","received_events_url":"https://api.github.com/users/ethervoid/received_events","type":"User","site_admin":false},"body":"Is there any possibility to get an error trying to force the encoding to UTF-8. Below we have a fallback to Win-1252.\r\n\r\nOther thing is that we should use constants instead of pain strings","created_at":"2015-11-20T08:26:58Z","updated_at":"2015-11-20T08:26:58Z","html_url":"https://github.com/CartoDB/cartodb/pull/6152#discussion_r45445150","pull_request_url":"https://api.github.com/repos/CartoDB/cartodb/pulls/6152","_links":{"self":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/comments/45445150"},"html":{"href":"https://github.com/CartoDB/cartodb/pull/6152#discussion_r45445150"},"pull_request":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/6152"}}},"pull_request":{"url":"https://api.github.com/repos/CartoDB/cartodb/pulls/6152","id":51114291,"html_url":"https://github.com/CartoDB/cartodb/pull/6152","diff_url":"https://github.com/CartoDB/cartodb/pull/6152.diff","patch_url":"https://github.com/CartoDB/cartodb/pull/6152.patch","issue_url":"https://api.github.com/repos/CartoDB/cartodb/issues/6152","number":6152,"state":"open","locked":false,"title":"Csv normalization 6116","user":{"login":"jgoizueta","id":5909,"avatar_url":"https://avatars.githubusercontent.com/u/5909?v=3","gravatar_id":"","url":"https://api.github.com/users/jgoizueta","html_url":"https://github.com/jgoizueta","followers_url":"https://api.github.com/users/jgoizueta/followers","following_url":"https://api.github.com/users/jgoizueta/following{/other_user}","gists_url":"https://api.github.com/users/jgoizueta/gists{/gist_id}","starred_url":"https://api.github.com/users/jgoizueta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jgoizueta/subscriptions","organizations_url":"https://api.github.com/users/jgoizueta/orgs","repos_url":"https://api.github.com/users/jgoizueta/repos","events_url":"https://api.github.com/users/jgoizueta/events{/privacy}","received_events_url":"https://api.github.com/users/jgoizueta/received_events","type":"User","site_admin":false},"body":"This introduces support for line breaks inside cells in CSV normalization and also makes CSV normalization more consistent with imports without normalization:\r\n\r\n* Row separators are detected and handle by the Ruby CSV module, which results in more consistency\r\n  with direct ogr2ogr handling (e.g. `\\r\\n` separators where incorrectly handle by the normalization)\r\n* Reverse line feeds in Windows-1252 encoding where previously removed for some reason\r\n  (see f54a953345a17be45336f9065257c1eb629db34e). Now we do as ogr2ogr and just preserve them.\r\n\r\nFixes #6116\r\n\r\nSome regression tests are added to specify the new desired behaviour.\r\n\r\nMalformed CSV lines are being silently ignored as was done before. This is ok for some cases in which such lines are blank (e.g. some trailing `\\n\\r\\n` found in some files). It would be nice to inform the user of non-blank dropped lines, so a counter has been introduced in the code to be used in some future feature.","created_at":"2015-11-18T17:22:01Z","updated_at":"2015-11-20T08:26:58Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1a5a9590a6ab7ee8621e500645afdfdbe60c30b0","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/CartoDB/cartodb/pulls/6152/commits","review_comments_url":"https://api.github.com/repos/CartoDB/cartodb/pulls/6152/comments","review_comment_url":"https://api.github.com/repos/CartoDB/cartodb/pulls/comments{/number}","comments_url":"https://api.github.com/repos/CartoDB/cartodb/issues/6152/comments","statuses_url":"https://api.github.com/repos/CartoDB/cartodb/statuses/3ad0ab575d43402b8311775216fe1f39c23724da","head":{"label":"CartoDB:csv_normalization_6116","ref":"csv_normalization_6116","sha":"3ad0ab575d43402b8311775216fe1f39c23724da","user":{"login":"CartoDB","id":1799254,"avatar_url":"https://avatars.githubusercontent.com/u/1799254?v=3","gravatar_id":"","url":"https://api.github.com/users/CartoDB","html_url":"https://github.com/CartoDB","followers_url":"https://api.github.com/users/CartoDB/followers","following_url":"https://api.github.com/users/CartoDB/following{/other_user}","gists_url":"https://api.github.com/users/CartoDB/gists{/gist_id}","starred_url":"https://api.github.com/users/CartoDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CartoDB/subscriptions","organizations_url":"https://api.github.com/users/CartoDB/orgs","repos_url":"https://api.github.com/users/CartoDB/repos","events_url":"https://api.github.com/users/CartoDB/events{/privacy}","received_events_url":"https://api.github.com/users/CartoDB/received_events","type":"Organization","site_admin":false},"repo":{"id":4909355,"name":"cartodb","full_name":"CartoDB/cartodb","owner":{"login":"CartoDB","id":1799254,"avatar_url":"https://avatars.githubusercontent.com/u/1799254?v=3","gravatar_id":"","url":"https://api.github.com/users/CartoDB","html_url":"https://github.com/CartoDB","followers_url":"https://api.github.com/users/CartoDB/followers","following_url":"https://api.github.com/users/CartoDB/following{/other_user}","gists_url":"https://api.github.com/users/CartoDB/gists{/gist_id}","starred_url":"https://api.github.com/users/CartoDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CartoDB/subscriptions","organizations_url":"https://api.github.com/users/CartoDB/orgs","repos_url":"https://api.github.com/users/CartoDB/repos","events_url":"https://api.github.com/users/CartoDB/events{/privacy}","received_events_url":"https://api.github.com/users/CartoDB/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CartoDB/cartodb","description":"Location Intelligence & Data Visualization tool","fork":false,"url":"https://api.github.com/repos/CartoDB/cartodb","forks_url":"https://api.github.com/repos/CartoDB/cartodb/forks","keys_url":"https://api.github.com/repos/CartoDB/cartodb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CartoDB/cartodb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CartoDB/cartodb/teams","hooks_url":"https://api.github.com/repos/CartoDB/cartodb/hooks","issue_events_url":"https://api.github.com/repos/CartoDB/cartodb/issues/events{/number}","events_url":"https://api.github.com/repos/CartoDB/cartodb/events","assignees_url":"https://api.github.com/repos/CartoDB/cartodb/assignees{/user}","branches_url":"https://api.github.com/repos/CartoDB/cartodb/branches{/branch}","tags_url":"https://api.github.com/repos/CartoDB/cartodb/tags","blobs_url":"https://api.github.com/repos/CartoDB/cartodb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CartoDB/cartodb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CartoDB/cartodb/git/refs{/sha}","trees_url":"https://api.github.com/repos/CartoDB/cartodb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CartoDB/cartodb/statuses/{sha}","languages_url":"https://api.github.com/repos/CartoDB/cartodb/languages","stargazers_url":"https://api.github.com/repos/CartoDB/cartodb/stargazers","contributors_url":"https://api.github.com/repos/CartoDB/cartodb/contributors","subscribers_url":"https://api.github.com/repos/CartoDB/cartodb/subscribers","subscription_url":"https://api.github.com/repos/CartoDB/cartodb/subscription","commits_url":"https://api.github.com/repos/CartoDB/cartodb/commits{/sha}","git_commits_url":"https://api.github.com/repos/CartoDB/cartodb/git/commits{/sha}","comments_url":"https://api.github.com/repos/CartoDB/cartodb/comments{/number}","issue_comment_url":"https://api.github.com/repos/CartoDB/cartodb/issues/comments{/number}","contents_url":"https://api.github.com/repos/CartoDB/cartodb/contents/{+path}","compare_url":"https://api.github.com/repos/CartoDB/cartodb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CartoDB/cartodb/merges","archive_url":"https://api.github.com/repos/CartoDB/cartodb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CartoDB/cartodb/downloads","issues_url":"https://api.github.com/repos/CartoDB/cartodb/issues{/number}","pulls_url":"https://api.github.com/repos/CartoDB/cartodb/pulls{/number}","milestones_url":"https://api.github.com/repos/CartoDB/cartodb/milestones{/number}","notifications_url":"https://api.github.com/repos/CartoDB/cartodb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CartoDB/cartodb/labels{/name}","releases_url":"https://api.github.com/repos/CartoDB/cartodb/releases{/id}","created_at":"2012-07-05T15:23:45Z","updated_at":"2015-11-18T23:25:44Z","pushed_at":"2015-11-19T20:29:02Z","git_url":"git://github.com/CartoDB/cartodb.git","ssh_url":"git@github.com:CartoDB/cartodb.git","clone_url":"https://github.com/CartoDB/cartodb.git","svn_url":"https://github.com/CartoDB/cartodb","homepage":"http://cartodb.com","size":314618,"stargazers_count":986,"watchers_count":986,"language":"HTML","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":234,"mirror_url":null,"open_issues_count":844,"forks":234,"open_issues":844,"watchers":986,"default_branch":"master"}},"base":{"label":"CartoDB:master","ref":"master","sha":"3267362f3a5a68b67755f0df51d52aa072342224","user":{"login":"CartoDB","id":1799254,"avatar_url":"https://avatars.githubusercontent.com/u/1799254?v=3","gravatar_id":"","url":"https://api.github.com/users/CartoDB","html_url":"https://github.com/CartoDB","followers_url":"https://api.github.com/users/CartoDB/followers","following_url":"https://api.github.com/users/CartoDB/following{/other_user}","gists_url":"https://api.github.com/users/CartoDB/gists{/gist_id}","starred_url":"https://api.github.com/users/CartoDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CartoDB/subscriptions","organizations_url":"https://api.github.com/users/CartoDB/orgs","repos_url":"https://api.github.com/users/CartoDB/repos","events_url":"https://api.github.com/users/CartoDB/events{/privacy}","received_events_url":"https://api.github.com/users/CartoDB/received_events","type":"Organization","site_admin":false},"repo":{"id":4909355,"name":"cartodb","full_name":"CartoDB/cartodb","owner":{"login":"CartoDB","id":1799254,"avatar_url":"https://avatars.githubusercontent.com/u/1799254?v=3","gravatar_id":"","url":"https://api.github.com/users/CartoDB","html_url":"https://github.com/CartoDB","followers_url":"https://api.github.com/users/CartoDB/followers","following_url":"https://api.github.com/users/CartoDB/following{/other_user}","gists_url":"https://api.github.com/users/CartoDB/gists{/gist_id}","starred_url":"https://api.github.com/users/CartoDB/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CartoDB/subscriptions","organizations_url":"https://api.github.com/users/CartoDB/orgs","repos_url":"https://api.github.com/users/CartoDB/repos","events_url":"https://api.github.com/users/CartoDB/events{/privacy}","received_events_url":"https://api.github.com/users/CartoDB/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CartoDB/cartodb","description":"Location Intelligence & Data Visualization tool","fork":false,"url":"https://api.github.com/repos/CartoDB/cartodb","forks_url":"https://api.github.com/repos/CartoDB/cartodb/forks","keys_url":"https://api.github.com/repos/CartoDB/cartodb/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CartoDB/cartodb/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CartoDB/cartodb/teams","hooks_url":"https://api.github.com/repos/CartoDB/cartodb/hooks","issue_events_url":"https://api.github.com/repos/CartoDB/cartodb/issues/events{/number}","events_url":"https://api.github.com/repos/CartoDB/cartodb/events","assignees_url":"https://api.github.com/repos/CartoDB/cartodb/assignees{/user}","branches_url":"https://api.github.com/repos/CartoDB/cartodb/branches{/branch}","tags_url":"https://api.github.com/repos/CartoDB/cartodb/tags","blobs_url":"https://api.github.com/repos/CartoDB/cartodb/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CartoDB/cartodb/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CartoDB/cartodb/git/refs{/sha}","trees_url":"https://api.github.com/repos/CartoDB/cartodb/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CartoDB/cartodb/statuses/{sha}","languages_url":"https://api.github.com/repos/CartoDB/cartodb/languages","stargazers_url":"https://api.github.com/repos/CartoDB/cartodb/stargazers","contributors_url":"https://api.github.com/repos/CartoDB/cartodb/contributors","subscribers_url":"https://api.github.com/repos/CartoDB/cartodb/subscribers","subscription_url":"https://api.github.com/repos/CartoDB/cartodb/subscription","commits_url":"https://api.github.com/repos/CartoDB/cartodb/commits{/sha}","git_commits_url":"https://api.github.com/repos/CartoDB/cartodb/git/commits{/sha}","comments_url":"https://api.github.com/repos/CartoDB/cartodb/comments{/number}","issue_comment_url":"https://api.github.com/repos/CartoDB/cartodb/issues/comments{/number}","contents_url":"https://api.github.com/repos/CartoDB/cartodb/contents/{+path}","compare_url":"https://api.github.com/repos/CartoDB/cartodb/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CartoDB/cartodb/merges","archive_url":"https://api.github.com/repos/CartoDB/cartodb/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CartoDB/cartodb/downloads","issues_url":"https://api.github.com/repos/CartoDB/cartodb/issues{/number}","pulls_url":"https://api.github.com/repos/CartoDB/cartodb/pulls{/number}","milestones_url":"https://api.github.com/repos/CartoDB/cartodb/milestones{/number}","notifications_url":"https://api.github.com/repos/CartoDB/cartodb/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CartoDB/cartodb/labels{/name}","releases_url":"https://api.github.com/repos/CartoDB/cartodb/releases{/id}","created_at":"2012-07-05T15:23:45Z","updated_at":"2015-11-18T23:25:44Z","pushed_at":"2015-11-19T20:29:02Z","git_url":"git://github.com/CartoDB/cartodb.git","ssh_url":"git@github.com:CartoDB/cartodb.git","clone_url":"https://github.com/CartoDB/cartodb.git","svn_url":"https://github.com/CartoDB/cartodb","homepage":"http://cartodb.com","size":314618,"stargazers_count":986,"watchers_count":986,"language":"HTML","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":234,"mirror_url":null,"open_issues_count":844,"forks":234,"open_issues":844,"watchers":986,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/6152"},"html":{"href":"https://github.com/CartoDB/cartodb/pull/6152"},"issue":{"href":"https://api.github.com/repos/CartoDB/cartodb/issues/6152"},"comments":{"href":"https://api.github.com/repos/CartoDB/cartodb/issues/6152/comments"},"review_comments":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/6152/comments"},"review_comment":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/CartoDB/cartodb/pulls/6152/commits"},"statuses":{"href":"https://api.github.com/repos/CartoDB/cartodb/statuses/3ad0ab575d43402b8311775216fe1f39c23724da"}}}}