{"action":"created","comment":{"url":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/comments/35225519","id":35225519,"diff_hunk":"@@ -73,41 +80,89 @@ def run_all(year_calage = 2011, year_data_list = [1995, 2000, 2005, 2011]):\n     build_depenses_homogenisees(year = year_data)\n     build_imputation_loyers_proprietaires(year = year_data)\n \n-    build_depenses_calees(year_calage, year_data)\n-    build_menage_consumption_by_categorie_fiscale(year_calage, year_data)\n+    build_depenses_calees(year_calage = year_calage, year_data = year_data)\n+    build_menage_consumption_by_categorie_fiscale(year_calage = year_calage, year_data = year_data)\n+\n     categorie_fiscale_data_frame = temporary_store[\"menage_consumption_by_categorie_fiscale_{}\".format(year_calage)]\n+    categorie_fiscale_data_frame.index = categorie_fiscale_data_frame.index.astype(ident_men_dtype)\n+\n+    temporary_store[\"menage_consumption_by_categorie_fiscale_{}\".format(year_calage)] = categorie_fiscale_data_frame\n+\n+    temporary_store[\"menage_consumption_by_categorie_fiscale_{}\".format(year_calage)] = categorie_fiscale_data_frame\n+\n     depenses_calees_by_grosposte = temporary_store[\"depenses_calees_by_grosposte_{}\".format(year_calage)]\n+    depenses_calees_by_grosposte.index = depenses_calees_by_grosposte.index.astype(ident_men_dtype)\n     depenses_calees = temporary_store[\"depenses_calees_{}\".format(year_calage)]\n+    depenses_calees.index = depenses_calees.index.astype(ident_men_dtype)\n \n     # Gestion des véhicules:\n     build_homogeneisation_vehicules(year = year_data)\n     if year_calage != 1995:\n         vehicule = temporary_store['automobile_{}'.format(year_data)]\n+        vehicule.index = vehicule.index.astype(ident_men_dtype)\n     else:\n         vehicule = None\n \n     # Gestion des variables socio démographiques:\n     build_homogeneisation_caracteristiques_sociales(year = year_data)\n     menage = temporary_store['donnes_socio_demog_{}'.format(year_data)]\n+    menage.index = menage.index.astype(ident_men_dtype)\n \n     # Gestion des variables revenus:\n     build_homogeneisation_revenus_menages(year = year_data)\n-    build_revenus_cales(year_calage, year_data)\n+    build_revenus_cales(year_calage = year_calage, year_data = year_data)\n     revenus = temporary_store[\"revenus_cales_{}\".format(year_calage)]\n+    revenus.index = revenus.index.astype(ident_men_dtype)\n \n     temporary_store.close()\n \n-    # DataFrame résultant de ces 4 étapes\n+    # Concaténation des résults de ces 4 étapes\n+\n+    preprocessed_data_frame_by_name = dict(\n+        revenus = revenus,\n+        vehicule = vehicule,\n+        categorie_fiscale_data_frame = categorie_fiscale_data_frame,\n+        menage = menage,\n+        depenses_calees = depenses_calees,\n+        depenses_calees_by_grosposte = depenses_calees_by_grosposte\n+        )\n+\n+    for name, preprocessed_data_frame in preprocessed_data_frame_by_name.iteritems():\n+        assert preprocessed_data_frame.index.name == 'ident_men', \\\n+            'Index is not labelled ident_men in data frame {}'.format(name)\n+        assert len(preprocessed_data_frame) != 0, 'Empty data frame {}'.format(name)\n+        print '-----'\n+        print name,\n+        print 'size: ', len(preprocessed_data_frame)\n+        print 'dtype :', preprocessed_data_frame.index.dtype,\n+        print 'nan_containing_variables: ', preprocessed_data_frame.isnull().any()[preprocessed_data_frame.isnull().any()].index\n+\n+        assert preprocessed_data_frame.index.dtype == numpy.dtype('O'), \"index for {} is {}\".format(\n+            name, preprocessed_data_frame.index.dtype)\n+\n     data_frame = pandas.concat(\n         [revenus, vehicule, categorie_fiscale_data_frame, menage, depenses_calees, depenses_calees_by_grosposte],\n         axis = 1,\n         )\n \n-    if year_data != 1995:\n-        data_frame.veh_tot = data_frame.veh_tot.fillna(0)\n-        data_frame.veh_essence = data_frame.veh_essence.fillna(0)\n-        data_frame.veh_diesel = data_frame.veh_diesel.fillna(0)\n-        data_frame.pourcentage_vehicule_essence = data_frame.pourcentage_vehicule_essence.fillna(0)\n+    nan_containing_variables = list(data_frame.isnull().any()[data_frame.isnull().any()].index)\n+    nan_containing_variables_by_name = dict(\n+        (name, list(set(nan_containing_variables).intersection(set(preprocessed_data_frame.columns))))\n+        for name, preprocessed_data_frame in preprocessed_data_frame_by_name.iteritems()\n+        )\n+    import pprint\n+\n+    pprint.pprint(nan_containing_variables_by_name)","path":"openfisca_france_data/input_data_builders/build_openfisca_indirect_taxation_survey_data/run_all.py","position":124,"original_position":124,"commit_id":"3c429df31a4de29bd0aeab6fccc5aeaf93807010","original_commit_id":"3c429df31a4de29bd0aeab6fccc5aeaf93807010","user":{"login":"benjello","id":1175927,"avatar_url":"https://avatars.githubusercontent.com/u/1175927?v=3","gravatar_id":"","url":"https://api.github.com/users/benjello","html_url":"https://github.com/benjello","followers_url":"https://api.github.com/users/benjello/followers","following_url":"https://api.github.com/users/benjello/following{/other_user}","gists_url":"https://api.github.com/users/benjello/gists{/gist_id}","starred_url":"https://api.github.com/users/benjello/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/benjello/subscriptions","organizations_url":"https://api.github.com/users/benjello/orgs","repos_url":"https://api.github.com/users/benjello/repos","events_url":"https://api.github.com/users/benjello/events{/privacy}","received_events_url":"https://api.github.com/users/benjello/received_events","type":"User","site_admin":false},"body":"Remove prints","created_at":"2015-07-22T15:26:18Z","updated_at":"2015-07-22T15:26:18Z","html_url":"https://github.com/openfisca/openfisca-france-data/pull/95#discussion_r35225519","pull_request_url":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/95","_links":{"self":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/comments/35225519"},"html":{"href":"https://github.com/openfisca/openfisca-france-data/pull/95#discussion_r35225519"},"pull_request":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/95"}}},"pull_request":{"url":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/95","id":37468733,"html_url":"https://github.com/openfisca/openfisca-france-data/pull/95","diff_url":"https://github.com/openfisca/openfisca-france-data/pull/95.diff","patch_url":"https://github.com/openfisca/openfisca-france-data/pull/95.patch","issue_url":"https://api.github.com/repos/openfisca/openfisca-france-data/issues/95","number":95,"state":"open","locked":false,"title":"Remplacement des Nan par des 0, changement des index de int à objets","user":{"login":"thomasdouenne","id":12607220,"avatar_url":"https://avatars.githubusercontent.com/u/12607220?v=3","gravatar_id":"","url":"https://api.github.com/users/thomasdouenne","html_url":"https://github.com/thomasdouenne","followers_url":"https://api.github.com/users/thomasdouenne/followers","following_url":"https://api.github.com/users/thomasdouenne/following{/other_user}","gists_url":"https://api.github.com/users/thomasdouenne/gists{/gist_id}","starred_url":"https://api.github.com/users/thomasdouenne/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thomasdouenne/subscriptions","organizations_url":"https://api.github.com/users/thomasdouenne/orgs","repos_url":"https://api.github.com/users/thomasdouenne/repos","events_url":"https://api.github.com/users/thomasdouenne/events{/privacy}","received_events_url":"https://api.github.com/users/thomasdouenne/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-06-11T14:24:41Z","updated_at":"2015-07-22T15:26:18Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f3528ed324ad7cdf60fc9c1d055122dda52de00f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/95/commits","review_comments_url":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/95/comments","review_comment_url":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/comments{/number}","comments_url":"https://api.github.com/repos/openfisca/openfisca-france-data/issues/95/comments","statuses_url":"https://api.github.com/repos/openfisca/openfisca-france-data/statuses/3c429df31a4de29bd0aeab6fccc5aeaf93807010","head":{"label":"thomasdouenne:dev","ref":"dev","sha":"3c429df31a4de29bd0aeab6fccc5aeaf93807010","user":{"login":"thomasdouenne","id":12607220,"avatar_url":"https://avatars.githubusercontent.com/u/12607220?v=3","gravatar_id":"","url":"https://api.github.com/users/thomasdouenne","html_url":"https://github.com/thomasdouenne","followers_url":"https://api.github.com/users/thomasdouenne/followers","following_url":"https://api.github.com/users/thomasdouenne/following{/other_user}","gists_url":"https://api.github.com/users/thomasdouenne/gists{/gist_id}","starred_url":"https://api.github.com/users/thomasdouenne/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thomasdouenne/subscriptions","organizations_url":"https://api.github.com/users/thomasdouenne/orgs","repos_url":"https://api.github.com/users/thomasdouenne/repos","events_url":"https://api.github.com/users/thomasdouenne/events{/privacy}","received_events_url":"https://api.github.com/users/thomasdouenne/received_events","type":"User","site_admin":false},"repo":{"id":36282011,"name":"openfisca-france-data","full_name":"thomasdouenne/openfisca-france-data","owner":{"login":"thomasdouenne","id":12607220,"avatar_url":"https://avatars.githubusercontent.com/u/12607220?v=3","gravatar_id":"","url":"https://api.github.com/users/thomasdouenne","html_url":"https://github.com/thomasdouenne","followers_url":"https://api.github.com/users/thomasdouenne/followers","following_url":"https://api.github.com/users/thomasdouenne/following{/other_user}","gists_url":"https://api.github.com/users/thomasdouenne/gists{/gist_id}","starred_url":"https://api.github.com/users/thomasdouenne/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thomasdouenne/subscriptions","organizations_url":"https://api.github.com/users/thomasdouenne/orgs","repos_url":"https://api.github.com/users/thomasdouenne/repos","events_url":"https://api.github.com/users/thomasdouenne/events{/privacy}","received_events_url":"https://api.github.com/users/thomasdouenne/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/thomasdouenne/openfisca-france-data","description":"","fork":true,"url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data","forks_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/forks","keys_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/keys{/key_id}","collaborators_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/teams","hooks_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/hooks","issue_events_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/issues/events{/number}","events_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/events","assignees_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/assignees{/user}","branches_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/branches{/branch}","tags_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/tags","blobs_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/git/refs{/sha}","trees_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/git/trees{/sha}","statuses_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/statuses/{sha}","languages_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/languages","stargazers_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/stargazers","contributors_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/contributors","subscribers_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/subscribers","subscription_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/subscription","commits_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/commits{/sha}","git_commits_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/git/commits{/sha}","comments_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/comments{/number}","issue_comment_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/issues/comments{/number}","contents_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/contents/{+path}","compare_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/compare/{base}...{head}","merges_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/merges","archive_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/downloads","issues_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/issues{/number}","pulls_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/pulls{/number}","milestones_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/milestones{/number}","notifications_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/labels{/name}","releases_url":"https://api.github.com/repos/thomasdouenne/openfisca-france-data/releases{/id}","created_at":"2015-05-26T08:20:09Z","updated_at":"2015-05-26T08:20:10Z","pushed_at":"2015-06-25T11:51:46Z","git_url":"git://github.com/thomasdouenne/openfisca-france-data.git","ssh_url":"git@github.com:thomasdouenne/openfisca-france-data.git","clone_url":"https://github.com/thomasdouenne/openfisca-france-data.git","svn_url":"https://github.com/thomasdouenne/openfisca-france-data","homepage":null,"size":3483,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"openfisca:dev","ref":"dev","sha":"b782f0985fcbd75d9e3c790b79d541456ed0adcd","user":{"login":"openfisca","id":1794404,"avatar_url":"https://avatars.githubusercontent.com/u/1794404?v=3","gravatar_id":"","url":"https://api.github.com/users/openfisca","html_url":"https://github.com/openfisca","followers_url":"https://api.github.com/users/openfisca/followers","following_url":"https://api.github.com/users/openfisca/following{/other_user}","gists_url":"https://api.github.com/users/openfisca/gists{/gist_id}","starred_url":"https://api.github.com/users/openfisca/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/openfisca/subscriptions","organizations_url":"https://api.github.com/users/openfisca/orgs","repos_url":"https://api.github.com/users/openfisca/repos","events_url":"https://api.github.com/users/openfisca/events{/privacy}","received_events_url":"https://api.github.com/users/openfisca/received_events","type":"Organization","site_admin":false},"repo":{"id":18373026,"name":"openfisca-france-data","full_name":"openfisca/openfisca-france-data","owner":{"login":"openfisca","id":1794404,"avatar_url":"https://avatars.githubusercontent.com/u/1794404?v=3","gravatar_id":"","url":"https://api.github.com/users/openfisca","html_url":"https://github.com/openfisca","followers_url":"https://api.github.com/users/openfisca/followers","following_url":"https://api.github.com/users/openfisca/following{/other_user}","gists_url":"https://api.github.com/users/openfisca/gists{/gist_id}","starred_url":"https://api.github.com/users/openfisca/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/openfisca/subscriptions","organizations_url":"https://api.github.com/users/openfisca/orgs","repos_url":"https://api.github.com/users/openfisca/repos","events_url":"https://api.github.com/users/openfisca/events{/privacy}","received_events_url":"https://api.github.com/users/openfisca/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/openfisca/openfisca-france-data","description":"","fork":false,"url":"https://api.github.com/repos/openfisca/openfisca-france-data","forks_url":"https://api.github.com/repos/openfisca/openfisca-france-data/forks","keys_url":"https://api.github.com/repos/openfisca/openfisca-france-data/keys{/key_id}","collaborators_url":"https://api.github.com/repos/openfisca/openfisca-france-data/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/openfisca/openfisca-france-data/teams","hooks_url":"https://api.github.com/repos/openfisca/openfisca-france-data/hooks","issue_events_url":"https://api.github.com/repos/openfisca/openfisca-france-data/issues/events{/number}","events_url":"https://api.github.com/repos/openfisca/openfisca-france-data/events","assignees_url":"https://api.github.com/repos/openfisca/openfisca-france-data/assignees{/user}","branches_url":"https://api.github.com/repos/openfisca/openfisca-france-data/branches{/branch}","tags_url":"https://api.github.com/repos/openfisca/openfisca-france-data/tags","blobs_url":"https://api.github.com/repos/openfisca/openfisca-france-data/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/openfisca/openfisca-france-data/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/openfisca/openfisca-france-data/git/refs{/sha}","trees_url":"https://api.github.com/repos/openfisca/openfisca-france-data/git/trees{/sha}","statuses_url":"https://api.github.com/repos/openfisca/openfisca-france-data/statuses/{sha}","languages_url":"https://api.github.com/repos/openfisca/openfisca-france-data/languages","stargazers_url":"https://api.github.com/repos/openfisca/openfisca-france-data/stargazers","contributors_url":"https://api.github.com/repos/openfisca/openfisca-france-data/contributors","subscribers_url":"https://api.github.com/repos/openfisca/openfisca-france-data/subscribers","subscription_url":"https://api.github.com/repos/openfisca/openfisca-france-data/subscription","commits_url":"https://api.github.com/repos/openfisca/openfisca-france-data/commits{/sha}","git_commits_url":"https://api.github.com/repos/openfisca/openfisca-france-data/git/commits{/sha}","comments_url":"https://api.github.com/repos/openfisca/openfisca-france-data/comments{/number}","issue_comment_url":"https://api.github.com/repos/openfisca/openfisca-france-data/issues/comments{/number}","contents_url":"https://api.github.com/repos/openfisca/openfisca-france-data/contents/{+path}","compare_url":"https://api.github.com/repos/openfisca/openfisca-france-data/compare/{base}...{head}","merges_url":"https://api.github.com/repos/openfisca/openfisca-france-data/merges","archive_url":"https://api.github.com/repos/openfisca/openfisca-france-data/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/openfisca/openfisca-france-data/downloads","issues_url":"https://api.github.com/repos/openfisca/openfisca-france-data/issues{/number}","pulls_url":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls{/number}","milestones_url":"https://api.github.com/repos/openfisca/openfisca-france-data/milestones{/number}","notifications_url":"https://api.github.com/repos/openfisca/openfisca-france-data/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/openfisca/openfisca-france-data/labels{/name}","releases_url":"https://api.github.com/repos/openfisca/openfisca-france-data/releases{/id}","created_at":"2014-04-02T16:25:09Z","updated_at":"2015-05-01T11:14:07Z","pushed_at":"2015-07-03T23:27:00Z","git_url":"git://github.com/openfisca/openfisca-france-data.git","ssh_url":"git@github.com:openfisca/openfisca-france-data.git","clone_url":"https://github.com/openfisca/openfisca-france-data.git","svn_url":"https://github.com/openfisca/openfisca-france-data","homepage":null,"size":4717,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":16,"mirror_url":null,"open_issues_count":4,"forks":16,"open_issues":4,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/95"},"html":{"href":"https://github.com/openfisca/openfisca-france-data/pull/95"},"issue":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/issues/95"},"comments":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/issues/95/comments"},"review_comments":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/95/comments"},"review_comment":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/pulls/95/commits"},"statuses":{"href":"https://api.github.com/repos/openfisca/openfisca-france-data/statuses/3c429df31a4de29bd0aeab6fccc5aeaf93807010"}}}}