{"action":"created","comment":{"url":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/comments/42326826","id":42326826,"diff_hunk":"@@ -0,0 +1,129 @@\n+# import findspark\n+# findspark.init()\n+import pyspark\n+import numpy as np\n+\n+sc = pyspark.SparkContext()\n+\n+N = 32\n+\n+def link_string_to_KV(s):\n+        src, dests = s.split(': ')\n+        dests = [int(to) for to in dests.split(' ')]\n+        return (int(src), dests)\n+\n+\n+def copartitioned(RDD1, RDD2):\n+    \"check if two RDDs are copartitioned\"\n+    return RDD1.partitioner == RDD2.partitioner\n+\n+def symmetric_graph(graph):\n+    \n+    lonely = graph.filter(lambda x: x[1] == [])\n+    a = graph.map(lambda x: [(x[0], i) for i in x[1]]).flatMap(lambda x: x)\n+    inverted = graph.map(lambda x: [(i,x[0]) for i in x[1]])\n+    b  = inverted.flatMap(lambda x: x)\n+    symmetric_graph = a.intersection(b).groupByKey().map(lambda x: (x[0], list(x[1]))).union(lonely)\n+    symmetric_graph = symmetric_graph.union(graph.subtractByKey(symmetric_graph).map(lambda x: (x[0], [])))\n+    \n+    return symmetric_graph\n+\n+def filterer(i):\n+    def filt(x):\n+        return x==itr_acc.value\n+    return filt\n+\n+def get_first_non_empty_cogroup(x2):\n+    for elem in x2:\n+        elem = list(elem)\n+        if len(elem) > 0:\n+            return elem[0]\n+\n+def BFS(graph, starting_node, N):\n+    \n+        #We set the starting node to have distance = 0 \n+    nodes = sc.parallelize([(starting_node, 1)]).partitionBy(N)\n+    \n+    graph.partitionBy(N).cache()\n+    \n+    total_char_acc = sc.accumulator(1)\n+    it_acc = sc.accumulator(0)\n+\n+    while True:\n+        it_acc += 1\n+        dist = it_acc.value\n+\n+        nodes_2 = nodes.filter(lambda x: filterer(x[1])).join(graph).flatMap(lambda x: x[1][1]).distinct().map(lambda x: (x, dist + 1)).partitionBy(N)\n+\n+        nodes = nodes.cogroup(nodes_2).map(lambda x: (x[0], get_first_non_empty_cogroup(list(x[1])))).partitionBy(N).cache()\n+        \n+        assert copartitioned(nodes_2, nodes)\n+\n+        if nodes.count() == total_char_acc.value:\n+            break\n+\n+        total_char_acc += nodes.count() - total_char_acc.value\n+        \n+    return nodes.count(), nodes\n+\n+\n+\n+def biggest_component(list_components):\n+    u, indices = np.unique(list_components, return_inverse=True)\n+    biggest = u[np.argmax(np.bincount(indices))]\n+    return list_components.count(biggest)\n+\n+def connected_components(graph, N):\n+    \n+    components = sc.parallelize([])\n+    n_components = sc.accumulator(0)\n+    n_nodes = graph.count()\n+    counted_nodes = 0\n+    \n+    while counted_nodes != n_nodes:\n+    \n+        component_index = n_components.value\n+        \n+        target_node = graph.keys().takeSample(False, 1, int(np.random.uniform(0,100000,1)))\n+                \n+        [nodes_added, touched_nodes] = BFS(graph, target_node[0], N)\n+                \n+        graph = graph.subtractByKey(touched_nodes).partitionBy(N).cache()\n+                \n+        touched_nodes = touched_nodes.map(lambda x: (x, component_index)).partitionBy(N)\n+        \n+        components = components.union(touched_nodes).partitionBy(N).cache()\n+        \n+        assert copartitioned(touched_nodes, components)\n+        \n+        n_components += 1\n+        counted_nodes = nodes_added + counted_nodes  \n+        print counted_nodes\n+        \n+    return biggest_component(components.map(lambda x: x[1]).collect()),  n_components.value\n+\n+\n+if __name__ == '__main__':\n+\n+\n+    links = sc.textFile('s3://Harvard-CS205/wikipedia/links-simple-sorted.txt')\n+    page_names = sc.textFile('s3://Harvard-CS205/wikipedia/titles-sorted.txt')\n+\n+    # links = sc.textFile('./testdata.txt')\n+    # page_names = sc.textFile('./pages.txt')\n+    # process links into (node #, [neighbor node #, neighbor node #, ...]\n+\n+    neighbor_graph = links.map(link_string_to_KV)\n+\n+    [maxC, nC] = connected_components(neighbor_graph, N)\n+\n+    neighbor_symmetric = symmetric_graph(neighbor_graph)\n+\n+    [maxC2, nC2] = connected_components(neighbor_symmetric, N)\n+\n+    print \"For the graph where each link is taken as symmetric, the total number of connected components is\", nC, \"and the biggest component has:\", maxC, \"elements.\"\n+\n+    print \"For the graph where two pages are only considered linked if they link to each other, the total number of connected components is\", nC2, \"and the biggest component has:\", maxC2, \"elements.\"\n+\n+\n+","path":"HW1/P5/P5_connected_components.py","position":129,"original_position":129,"commit_id":"1fcfb6e8f396efbeb567d9ffb57a9f9599dc0413","original_commit_id":"1fcfb6e8f396efbeb567d9ffb57a9f9599dc0413","user":{"login":"matheuscfernandes","id":8809429,"avatar_url":"https://avatars.githubusercontent.com/u/8809429?v=3","gravatar_id":"","url":"https://api.github.com/users/matheuscfernandes","html_url":"https://github.com/matheuscfernandes","followers_url":"https://api.github.com/users/matheuscfernandes/followers","following_url":"https://api.github.com/users/matheuscfernandes/following{/other_user}","gists_url":"https://api.github.com/users/matheuscfernandes/gists{/gist_id}","starred_url":"https://api.github.com/users/matheuscfernandes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matheuscfernandes/subscriptions","organizations_url":"https://api.github.com/users/matheuscfernandes/orgs","repos_url":"https://api.github.com/users/matheuscfernandes/repos","events_url":"https://api.github.com/users/matheuscfernandes/events{/privacy}","received_events_url":"https://api.github.com/users/matheuscfernandes/received_events","type":"User","site_admin":false},"body":"Did this actually work for you? If so, that is awesome! I could not figure this problem out.","created_at":"2015-10-18T21:38:34Z","updated_at":"2015-10-18T21:38:34Z","html_url":"https://github.com/harvard-cs205/cs205-homework/pull/231#discussion_r42326826","pull_request_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/231","_links":{"self":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/comments/42326826"},"html":{"href":"https://github.com/harvard-cs205/cs205-homework/pull/231#discussion_r42326826"},"pull_request":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/231"}}},"pull_request":{"url":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/231","id":46742080,"html_url":"https://github.com/harvard-cs205/cs205-homework/pull/231","diff_url":"https://github.com/harvard-cs205/cs205-homework/pull/231.diff","patch_url":"https://github.com/harvard-cs205/cs205-homework/pull/231.patch","issue_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/issues/231","number":231,"state":"open","locked":false,"title":"Hw1","user":{"login":"isadoranun","id":8644361,"avatar_url":"https://avatars.githubusercontent.com/u/8644361?v=3","gravatar_id":"","url":"https://api.github.com/users/isadoranun","html_url":"https://github.com/isadoranun","followers_url":"https://api.github.com/users/isadoranun/followers","following_url":"https://api.github.com/users/isadoranun/following{/other_user}","gists_url":"https://api.github.com/users/isadoranun/gists{/gist_id}","starred_url":"https://api.github.com/users/isadoranun/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/isadoranun/subscriptions","organizations_url":"https://api.github.com/users/isadoranun/orgs","repos_url":"https://api.github.com/users/isadoranun/repos","events_url":"https://api.github.com/users/isadoranun/events{/privacy}","received_events_url":"https://api.github.com/users/isadoranun/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-10-05T03:32:35Z","updated_at":"2015-10-18T21:38:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"635edf933eff79c3d067bda2a865c2789fb3a77d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/231/commits","review_comments_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/231/comments","review_comment_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/comments{/number}","comments_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/issues/231/comments","statuses_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/statuses/1fcfb6e8f396efbeb567d9ffb57a9f9599dc0413","head":{"label":"isadoranun:HW1","ref":"HW1","sha":"1fcfb6e8f396efbeb567d9ffb57a9f9599dc0413","user":{"login":"isadoranun","id":8644361,"avatar_url":"https://avatars.githubusercontent.com/u/8644361?v=3","gravatar_id":"","url":"https://api.github.com/users/isadoranun","html_url":"https://github.com/isadoranun","followers_url":"https://api.github.com/users/isadoranun/followers","following_url":"https://api.github.com/users/isadoranun/following{/other_user}","gists_url":"https://api.github.com/users/isadoranun/gists{/gist_id}","starred_url":"https://api.github.com/users/isadoranun/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/isadoranun/subscriptions","organizations_url":"https://api.github.com/users/isadoranun/orgs","repos_url":"https://api.github.com/users/isadoranun/repos","events_url":"https://api.github.com/users/isadoranun/events{/privacy}","received_events_url":"https://api.github.com/users/isadoranun/received_events","type":"User","site_admin":false},"repo":{"id":42022953,"name":"cs205-homework","full_name":"isadoranun/cs205-homework","owner":{"login":"isadoranun","id":8644361,"avatar_url":"https://avatars.githubusercontent.com/u/8644361?v=3","gravatar_id":"","url":"https://api.github.com/users/isadoranun","html_url":"https://github.com/isadoranun","followers_url":"https://api.github.com/users/isadoranun/followers","following_url":"https://api.github.com/users/isadoranun/following{/other_user}","gists_url":"https://api.github.com/users/isadoranun/gists{/gist_id}","starred_url":"https://api.github.com/users/isadoranun/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/isadoranun/subscriptions","organizations_url":"https://api.github.com/users/isadoranun/orgs","repos_url":"https://api.github.com/users/isadoranun/repos","events_url":"https://api.github.com/users/isadoranun/events{/privacy}","received_events_url":"https://api.github.com/users/isadoranun/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/isadoranun/cs205-homework","description":"","fork":true,"url":"https://api.github.com/repos/isadoranun/cs205-homework","forks_url":"https://api.github.com/repos/isadoranun/cs205-homework/forks","keys_url":"https://api.github.com/repos/isadoranun/cs205-homework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/isadoranun/cs205-homework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/isadoranun/cs205-homework/teams","hooks_url":"https://api.github.com/repos/isadoranun/cs205-homework/hooks","issue_events_url":"https://api.github.com/repos/isadoranun/cs205-homework/issues/events{/number}","events_url":"https://api.github.com/repos/isadoranun/cs205-homework/events","assignees_url":"https://api.github.com/repos/isadoranun/cs205-homework/assignees{/user}","branches_url":"https://api.github.com/repos/isadoranun/cs205-homework/branches{/branch}","tags_url":"https://api.github.com/repos/isadoranun/cs205-homework/tags","blobs_url":"https://api.github.com/repos/isadoranun/cs205-homework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/isadoranun/cs205-homework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/isadoranun/cs205-homework/git/refs{/sha}","trees_url":"https://api.github.com/repos/isadoranun/cs205-homework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/isadoranun/cs205-homework/statuses/{sha}","languages_url":"https://api.github.com/repos/isadoranun/cs205-homework/languages","stargazers_url":"https://api.github.com/repos/isadoranun/cs205-homework/stargazers","contributors_url":"https://api.github.com/repos/isadoranun/cs205-homework/contributors","subscribers_url":"https://api.github.com/repos/isadoranun/cs205-homework/subscribers","subscription_url":"https://api.github.com/repos/isadoranun/cs205-homework/subscription","commits_url":"https://api.github.com/repos/isadoranun/cs205-homework/commits{/sha}","git_commits_url":"https://api.github.com/repos/isadoranun/cs205-homework/git/commits{/sha}","comments_url":"https://api.github.com/repos/isadoranun/cs205-homework/comments{/number}","issue_comment_url":"https://api.github.com/repos/isadoranun/cs205-homework/issues/comments{/number}","contents_url":"https://api.github.com/repos/isadoranun/cs205-homework/contents/{+path}","compare_url":"https://api.github.com/repos/isadoranun/cs205-homework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/isadoranun/cs205-homework/merges","archive_url":"https://api.github.com/repos/isadoranun/cs205-homework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/isadoranun/cs205-homework/downloads","issues_url":"https://api.github.com/repos/isadoranun/cs205-homework/issues{/number}","pulls_url":"https://api.github.com/repos/isadoranun/cs205-homework/pulls{/number}","milestones_url":"https://api.github.com/repos/isadoranun/cs205-homework/milestones{/number}","notifications_url":"https://api.github.com/repos/isadoranun/cs205-homework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/isadoranun/cs205-homework/labels{/name}","releases_url":"https://api.github.com/repos/isadoranun/cs205-homework/releases{/id}","created_at":"2015-09-06T23:59:05Z","updated_at":"2015-09-06T23:59:05Z","pushed_at":"2015-10-06T01:11:12Z","git_url":"git://github.com/isadoranun/cs205-homework.git","ssh_url":"git@github.com:isadoranun/cs205-homework.git","clone_url":"https://github.com/isadoranun/cs205-homework.git","svn_url":"https://github.com/isadoranun/cs205-homework","homepage":null,"size":3286,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"harvard-cs205:master","ref":"master","sha":"406f864b38a6c8a3d89cba66cb5228e98907bab7","user":{"login":"harvard-cs205","id":13456002,"avatar_url":"https://avatars.githubusercontent.com/u/13456002?v=3","gravatar_id":"","url":"https://api.github.com/users/harvard-cs205","html_url":"https://github.com/harvard-cs205","followers_url":"https://api.github.com/users/harvard-cs205/followers","following_url":"https://api.github.com/users/harvard-cs205/following{/other_user}","gists_url":"https://api.github.com/users/harvard-cs205/gists{/gist_id}","starred_url":"https://api.github.com/users/harvard-cs205/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/harvard-cs205/subscriptions","organizations_url":"https://api.github.com/users/harvard-cs205/orgs","repos_url":"https://api.github.com/users/harvard-cs205/repos","events_url":"https://api.github.com/users/harvard-cs205/events{/privacy}","received_events_url":"https://api.github.com/users/harvard-cs205/received_events","type":"Organization","site_admin":false},"repo":{"id":40614104,"name":"cs205-homework","full_name":"harvard-cs205/cs205-homework","owner":{"login":"harvard-cs205","id":13456002,"avatar_url":"https://avatars.githubusercontent.com/u/13456002?v=3","gravatar_id":"","url":"https://api.github.com/users/harvard-cs205","html_url":"https://github.com/harvard-cs205","followers_url":"https://api.github.com/users/harvard-cs205/followers","following_url":"https://api.github.com/users/harvard-cs205/following{/other_user}","gists_url":"https://api.github.com/users/harvard-cs205/gists{/gist_id}","starred_url":"https://api.github.com/users/harvard-cs205/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/harvard-cs205/subscriptions","organizations_url":"https://api.github.com/users/harvard-cs205/orgs","repos_url":"https://api.github.com/users/harvard-cs205/repos","events_url":"https://api.github.com/users/harvard-cs205/events{/privacy}","received_events_url":"https://api.github.com/users/harvard-cs205/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/harvard-cs205/cs205-homework","description":"","fork":false,"url":"https://api.github.com/repos/harvard-cs205/cs205-homework","forks_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/forks","keys_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/teams","hooks_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/hooks","issue_events_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/issues/events{/number}","events_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/events","assignees_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/assignees{/user}","branches_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/branches{/branch}","tags_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/tags","blobs_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/git/refs{/sha}","trees_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/statuses/{sha}","languages_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/languages","stargazers_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/stargazers","contributors_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/contributors","subscribers_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/subscribers","subscription_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/subscription","commits_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/commits{/sha}","git_commits_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/git/commits{/sha}","comments_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/comments{/number}","issue_comment_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/issues/comments{/number}","contents_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/contents/{+path}","compare_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/merges","archive_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/downloads","issues_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/issues{/number}","pulls_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls{/number}","milestones_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/milestones{/number}","notifications_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/labels{/name}","releases_url":"https://api.github.com/repos/harvard-cs205/cs205-homework/releases{/id}","created_at":"2015-08-12T17:18:15Z","updated_at":"2015-10-16T14:48:16Z","pushed_at":"2015-10-17T02:44:38Z","git_url":"git://github.com/harvard-cs205/cs205-homework.git","ssh_url":"git@github.com:harvard-cs205/cs205-homework.git","clone_url":"https://github.com/harvard-cs205/cs205-homework.git","svn_url":"https://github.com/harvard-cs205/cs205-homework","homepage":null,"size":81042,"stargazers_count":4,"watchers_count":4,"language":"C","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":99,"mirror_url":null,"open_issues_count":265,"forks":99,"open_issues":265,"watchers":4,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/231"},"html":{"href":"https://github.com/harvard-cs205/cs205-homework/pull/231"},"issue":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/issues/231"},"comments":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/issues/231/comments"},"review_comments":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/231/comments"},"review_comment":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/pulls/231/commits"},"statuses":{"href":"https://api.github.com/repos/harvard-cs205/cs205-homework/statuses/1fcfb6e8f396efbeb567d9ffb57a9f9599dc0413"}}}}