{"action":"created","comment":{"url":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/comments/45306695","id":45306695,"diff_hunk":"@@ -0,0 +1,278 @@\n+# Copyright (C) 2015 Swift Navigation Inc.\n+#\n+# This source is subject to the license found in the file 'LICENSE' which must\n+# be be distributed together with this source. All other rights reserved.\n+#\n+# THIS CODE AND INFORMATION IS PROVIDED \"AS IS\" WITHOUT WARRANTY OF ANY KIND,\n+# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED\n+# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE.\n+\n+\"\"\"\n+simulate.py\n+\n+Contains a set of utility functions that help convert log messages\n+to and from sbp C-type classes and pandas DataFrames.\n+\"\"\"\n+\n+import os\n+import xray\n+import datetime\n+import itertools\n+import numpy as np\n+import pandas as pd\n+\n+import sbp.piksi as piksi\n+import sbp.logging as lg\n+import sbp.tracking as tr\n+import sbp.navigation as nav\n+import sbp.acquisition as acq\n+import sbp.observation as ob\n+\n+from sbp.utils import exclude_fields, walk_json_dict\n+from swiftnav.gpstime import gpst_components2datetime\n+\n+import gnss_analysis.constants as c\n+\n+obs_index = pd.MultiIndex.from_product([['rover', 'base'], np.arange(32)],\n+                                   names=['source', 'sid'])\n+\n+\n+def time_from_message(wn, tow):\n+  \"\"\"\n+  Takes a message with a time field stored as week number and time\n+  of week and converts it to a TimeStamp.\n+  \"\"\"\n+  # compute the time from the week number and time of week\n+  time = gpst_components2datetime(wn, tow / c.MSEC_TO_SECONDS)\n+  return time\n+\n+\n+def get_source(msg):\n+  \"\"\"\n+  Determines if a message originated at the rover or base station.\n+  \"\"\"\n+  # if the source field is 0 the source of observations is\n+  # from the base, otherwise  the rover.\n+  return 'rover' if msg.sender else 'base'\n+\n+\n+def observation_to_dataframe(msg, data):\n+  \"\"\"\n+  Convert an observation message to a dataframe which\n+  is indexed by the msg source and satellite id.\n+\n+  Parameters\n+  ----------\n+  msg : ob.MsgObs or ob.MsgObsDepA\n+    An sbp ephemeris message.\n+  data :\n+    The data corresponding to the msg.\n+\n+  Returns\n+  -------\n+  df : pd.DataFrame\n+    A DataFrame holding all the observations from various\n+    satellites.  The DataFrame is indexed by source and\n+    satellite, ('rover', 16) for example.  The DataFrame\n+    holds pseudorange, carrier phase\n+  \"\"\"\n+  # make sure the message is an observation message\n+  assert isinstance(msg, (ob.MsgObs, ob.MsgObsDepA))\n+  # determine the source\n+  source = get_source(msg)\n+\n+  def satellite_id(obs):\n+    return obs.sid if msg.msg_type is ob.SBP_MSG_OBS else obs.prn\n+\n+  # determine the time from week number and time of week\n+  time = time_from_message(msg.header.t.wn, msg.header.t.tow)\n+  timestamp = (time - datetime.datetime(1970, 1, 1)).total_seconds()\n+\n+  def extract_observations(obs):\n+      # Convert pseudorange, carrier phase to SI units.\n+      v = {'P': obs.P / c.CM_TO_M,\n+           'L': obs.L.i + obs.L.f / c.Q32_WIDTH,\n+           'cn0': obs.cn0,\n+           'lock': obs.lock,\n+           'host_time': data['timestamp'],\n+           'host_offset': data['delta'],\n+           'timestamp': timestamp,\n+           }\n+      return v\n+\n+  # Compute the indices of each of the observations\n+  source_sid_pairs = [(source, satellite_id(o)) for o in msg.obs]\n+  # assemble into a Multiindex\n+  idx = pd.MultiIndex.from_tuples(source_sid_pairs,\n+                                  names=['source', 'sid'])\n+  # Combine into a dataframe.\n+  df = pd.DataFrame([extract_observations(o) for o in msg.obs],\n+                    index=idx)\n+  return df\n+\n+\n+def ephemeris_to_dataframe(msg, data):\n+  \"\"\"\n+  Converts an ephermesis log message to a pandas Dataframe\n+  that is indexed by the source and satellite id.\n+\n+  Parameters\n+  ----------\n+  msg : ob.MsgEphemeris, ob.MsgEphemerisDepA ob.MsgEphemerisDepB\n+    An sbp ephemeris message.\n+  data :\n+    The data corresponding to the msg.\n+\n+  Returns\n+  -------\n+  eph : pd.DataFrame\n+    A DataFrame indexed by the source and satellite id, ('rover', 16)\n+    for example.  The columns then hold the ephemeris parameters.\n+  \"\"\"\n+  assert isinstance(msg, (ob.MsgEphemeris,","path":"gnss_analysis/simulate.py","position":132,"original_position":132,"commit_id":"6ce8aa4b065f71d415ae8f2412fbc9fb6348c8d2","original_commit_id":"6ce8aa4b065f71d415ae8f2412fbc9fb6348c8d2","user":{"login":"mookerji","id":353255,"avatar_url":"https://avatars.githubusercontent.com/u/353255?v=3","gravatar_id":"","url":"https://api.github.com/users/mookerji","html_url":"https://github.com/mookerji","followers_url":"https://api.github.com/users/mookerji/followers","following_url":"https://api.github.com/users/mookerji/following{/other_user}","gists_url":"https://api.github.com/users/mookerji/gists{/gist_id}","starred_url":"https://api.github.com/users/mookerji/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mookerji/subscriptions","organizations_url":"https://api.github.com/users/mookerji/orgs","repos_url":"https://api.github.com/users/mookerji/repos","events_url":"https://api.github.com/users/mookerji/events{/privacy}","received_events_url":"https://api.github.com/users/mookerji/received_events","type":"User","site_admin":false},"body":"Note, at this point, you can probably stop supporting the deprecated messages.","created_at":"2015-11-19T06:32:36Z","updated_at":"2015-11-19T06:32:36Z","html_url":"https://github.com/swift-nav/gnss-analysis/pull/70#discussion_r45306695","pull_request_url":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/70","_links":{"self":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/comments/45306695"},"html":{"href":"https://github.com/swift-nav/gnss-analysis/pull/70#discussion_r45306695"},"pull_request":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/70"}}},"pull_request":{"url":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/70","id":51158522,"html_url":"https://github.com/swift-nav/gnss-analysis/pull/70","diff_url":"https://github.com/swift-nav/gnss-analysis/pull/70.diff","patch_url":"https://github.com/swift-nav/gnss-analysis/pull/70.patch","issue_url":"https://api.github.com/repos/swift-nav/gnss-analysis/issues/70","number":70,"state":"open","locked":false,"title":"Alternative simulation based approach to log processing.","user":{"login":"akleeman","id":514053,"avatar_url":"https://avatars.githubusercontent.com/u/514053?v=3","gravatar_id":"","url":"https://api.github.com/users/akleeman","html_url":"https://github.com/akleeman","followers_url":"https://api.github.com/users/akleeman/followers","following_url":"https://api.github.com/users/akleeman/following{/other_user}","gists_url":"https://api.github.com/users/akleeman/gists{/gist_id}","starred_url":"https://api.github.com/users/akleeman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akleeman/subscriptions","organizations_url":"https://api.github.com/users/akleeman/orgs","repos_url":"https://api.github.com/users/akleeman/repos","events_url":"https://api.github.com/users/akleeman/events{/privacy}","received_events_url":"https://api.github.com/users/akleeman/received_events","type":"User","site_admin":false},"body":"Added simulate.py which processes log files on the fly and simulates the chip state at each iteration.\r\n\r\nThe function of most interest here is `simulate.simulate_from_log` which produces DataFrames that\r\nhold the current observations and ephemerides, similar to the output of records2table and fill_obs. \r\n\r\nHere is an example of what the 'state' looks like at a particular iteration:\r\n```\r\n> states = simulate.simulate_from_log(log)\r\n> print state.next()\r\n                       L            P       af0           af1  af2  \\\r\nsource sid                                                           \r\nbase   0   -68796.199219  21942167.62       NaN           NaN  NaN   \r\n       8    66616.105469  22980000.00       NaN           NaN  NaN   \r\n       22   55895.683594  20915355.02       NaN           NaN  NaN   \r\n       25   47762.078125  21413257.09       NaN           NaN  NaN   \r\n       30  -68259.582031  21359479.92       NaN           NaN  NaN   \r\n       31  -35515.554688  20420485.33       NaN           NaN  NaN   \r\nrover  15   95840.402344  22980000.00 -0.000132  3.637979e-12    0   \r\n       22   47298.339844  21640246.07 -0.000106 -2.501110e-12    0   \r\n       25   42062.238281  22138143.08  0.000027  3.751666e-12    0   \r\n       30  -73294.644531  22084550.17  0.000317 -1.023182e-12    0   \r\n```\r\n\r\nThe results could be stored as an HDF5 file,\r\n```\r\ndef log_to_hdf5(log_file):\r\n    with JSONLogIterator(log_file) as log:\r\n      states = simulate.simulate_from_log(log)\r\n      data = pd.concat(simulate.simulate_from_log(log))\r\n    data.to_hdf('test.hdf', 'x', mode='w', complevel=9, complib='zlib')\r\n```\r\nor used to produce on the fly position estimates,\r\n```\r\ndef estimate_position(state):\r\n    c_objects = create_c_objects_from_state(state)\r\n    return estimate_position_using_libswiftnav(c_objects)\r\n\r\npositions = (estimate_position(state) for state in simulate.simulate_from_log(log))\r\n```\r\n\r\nWorth noting that the resulting HDF5 files would have a very different format from the current format ... which could cause some backward compatibility issues, though (I think?) the outputs from records2table and fill_obs aren't used in production?\r\n\r\nThis code is functional on a single test example ... if it looks promising I can flush out the tests.\r\n\r\n@mookerji @fnoble","created_at":"2015-11-18T23:24:41Z","updated_at":"2015-11-19T06:32:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0017a2164b6b6f6397fc494ef3594b69c474f1b2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/70/commits","review_comments_url":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/70/comments","review_comment_url":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/comments{/number}","comments_url":"https://api.github.com/repos/swift-nav/gnss-analysis/issues/70/comments","statuses_url":"https://api.github.com/repos/swift-nav/gnss-analysis/statuses/6ce8aa4b065f71d415ae8f2412fbc9fb6348c8d2","head":{"label":"akleeman:alternate_log_processing","ref":"alternate_log_processing","sha":"6ce8aa4b065f71d415ae8f2412fbc9fb6348c8d2","user":{"login":"akleeman","id":514053,"avatar_url":"https://avatars.githubusercontent.com/u/514053?v=3","gravatar_id":"","url":"https://api.github.com/users/akleeman","html_url":"https://github.com/akleeman","followers_url":"https://api.github.com/users/akleeman/followers","following_url":"https://api.github.com/users/akleeman/following{/other_user}","gists_url":"https://api.github.com/users/akleeman/gists{/gist_id}","starred_url":"https://api.github.com/users/akleeman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akleeman/subscriptions","organizations_url":"https://api.github.com/users/akleeman/orgs","repos_url":"https://api.github.com/users/akleeman/repos","events_url":"https://api.github.com/users/akleeman/events{/privacy}","received_events_url":"https://api.github.com/users/akleeman/received_events","type":"User","site_admin":false},"repo":{"id":45875237,"name":"gnss-analysis","full_name":"akleeman/gnss-analysis","owner":{"login":"akleeman","id":514053,"avatar_url":"https://avatars.githubusercontent.com/u/514053?v=3","gravatar_id":"","url":"https://api.github.com/users/akleeman","html_url":"https://github.com/akleeman","followers_url":"https://api.github.com/users/akleeman/followers","following_url":"https://api.github.com/users/akleeman/following{/other_user}","gists_url":"https://api.github.com/users/akleeman/gists{/gist_id}","starred_url":"https://api.github.com/users/akleeman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akleeman/subscriptions","organizations_url":"https://api.github.com/users/akleeman/orgs","repos_url":"https://api.github.com/users/akleeman/repos","events_url":"https://api.github.com/users/akleeman/events{/privacy}","received_events_url":"https://api.github.com/users/akleeman/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/akleeman/gnss-analysis","description":"Tools for analyzing GNSS algorithms and GNSS data","fork":true,"url":"https://api.github.com/repos/akleeman/gnss-analysis","forks_url":"https://api.github.com/repos/akleeman/gnss-analysis/forks","keys_url":"https://api.github.com/repos/akleeman/gnss-analysis/keys{/key_id}","collaborators_url":"https://api.github.com/repos/akleeman/gnss-analysis/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/akleeman/gnss-analysis/teams","hooks_url":"https://api.github.com/repos/akleeman/gnss-analysis/hooks","issue_events_url":"https://api.github.com/repos/akleeman/gnss-analysis/issues/events{/number}","events_url":"https://api.github.com/repos/akleeman/gnss-analysis/events","assignees_url":"https://api.github.com/repos/akleeman/gnss-analysis/assignees{/user}","branches_url":"https://api.github.com/repos/akleeman/gnss-analysis/branches{/branch}","tags_url":"https://api.github.com/repos/akleeman/gnss-analysis/tags","blobs_url":"https://api.github.com/repos/akleeman/gnss-analysis/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/akleeman/gnss-analysis/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/akleeman/gnss-analysis/git/refs{/sha}","trees_url":"https://api.github.com/repos/akleeman/gnss-analysis/git/trees{/sha}","statuses_url":"https://api.github.com/repos/akleeman/gnss-analysis/statuses/{sha}","languages_url":"https://api.github.com/repos/akleeman/gnss-analysis/languages","stargazers_url":"https://api.github.com/repos/akleeman/gnss-analysis/stargazers","contributors_url":"https://api.github.com/repos/akleeman/gnss-analysis/contributors","subscribers_url":"https://api.github.com/repos/akleeman/gnss-analysis/subscribers","subscription_url":"https://api.github.com/repos/akleeman/gnss-analysis/subscription","commits_url":"https://api.github.com/repos/akleeman/gnss-analysis/commits{/sha}","git_commits_url":"https://api.github.com/repos/akleeman/gnss-analysis/git/commits{/sha}","comments_url":"https://api.github.com/repos/akleeman/gnss-analysis/comments{/number}","issue_comment_url":"https://api.github.com/repos/akleeman/gnss-analysis/issues/comments{/number}","contents_url":"https://api.github.com/repos/akleeman/gnss-analysis/contents/{+path}","compare_url":"https://api.github.com/repos/akleeman/gnss-analysis/compare/{base}...{head}","merges_url":"https://api.github.com/repos/akleeman/gnss-analysis/merges","archive_url":"https://api.github.com/repos/akleeman/gnss-analysis/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/akleeman/gnss-analysis/downloads","issues_url":"https://api.github.com/repos/akleeman/gnss-analysis/issues{/number}","pulls_url":"https://api.github.com/repos/akleeman/gnss-analysis/pulls{/number}","milestones_url":"https://api.github.com/repos/akleeman/gnss-analysis/milestones{/number}","notifications_url":"https://api.github.com/repos/akleeman/gnss-analysis/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/akleeman/gnss-analysis/labels{/name}","releases_url":"https://api.github.com/repos/akleeman/gnss-analysis/releases{/id}","created_at":"2015-11-09T23:58:44Z","updated_at":"2015-11-09T23:58:46Z","pushed_at":"2015-11-18T23:06:00Z","git_url":"git://github.com/akleeman/gnss-analysis.git","ssh_url":"git@github.com:akleeman/gnss-analysis.git","clone_url":"https://github.com/akleeman/gnss-analysis.git","svn_url":"https://github.com/akleeman/gnss-analysis","homepage":"","size":7594,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"swift-nav:master","ref":"master","sha":"d978ac2ba1b586c1f28bf1099e5715c02d6d508d","user":{"login":"swift-nav","id":1069835,"avatar_url":"https://avatars.githubusercontent.com/u/1069835?v=3","gravatar_id":"","url":"https://api.github.com/users/swift-nav","html_url":"https://github.com/swift-nav","followers_url":"https://api.github.com/users/swift-nav/followers","following_url":"https://api.github.com/users/swift-nav/following{/other_user}","gists_url":"https://api.github.com/users/swift-nav/gists{/gist_id}","starred_url":"https://api.github.com/users/swift-nav/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/swift-nav/subscriptions","organizations_url":"https://api.github.com/users/swift-nav/orgs","repos_url":"https://api.github.com/users/swift-nav/repos","events_url":"https://api.github.com/users/swift-nav/events{/privacy}","received_events_url":"https://api.github.com/users/swift-nav/received_events","type":"Organization","site_admin":false},"repo":{"id":19047568,"name":"gnss-analysis","full_name":"swift-nav/gnss-analysis","owner":{"login":"swift-nav","id":1069835,"avatar_url":"https://avatars.githubusercontent.com/u/1069835?v=3","gravatar_id":"","url":"https://api.github.com/users/swift-nav","html_url":"https://github.com/swift-nav","followers_url":"https://api.github.com/users/swift-nav/followers","following_url":"https://api.github.com/users/swift-nav/following{/other_user}","gists_url":"https://api.github.com/users/swift-nav/gists{/gist_id}","starred_url":"https://api.github.com/users/swift-nav/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/swift-nav/subscriptions","organizations_url":"https://api.github.com/users/swift-nav/orgs","repos_url":"https://api.github.com/users/swift-nav/repos","events_url":"https://api.github.com/users/swift-nav/events{/privacy}","received_events_url":"https://api.github.com/users/swift-nav/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/swift-nav/gnss-analysis","description":"Tools for analyzing GNSS algorithms and GNSS data","fork":false,"url":"https://api.github.com/repos/swift-nav/gnss-analysis","forks_url":"https://api.github.com/repos/swift-nav/gnss-analysis/forks","keys_url":"https://api.github.com/repos/swift-nav/gnss-analysis/keys{/key_id}","collaborators_url":"https://api.github.com/repos/swift-nav/gnss-analysis/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/swift-nav/gnss-analysis/teams","hooks_url":"https://api.github.com/repos/swift-nav/gnss-analysis/hooks","issue_events_url":"https://api.github.com/repos/swift-nav/gnss-analysis/issues/events{/number}","events_url":"https://api.github.com/repos/swift-nav/gnss-analysis/events","assignees_url":"https://api.github.com/repos/swift-nav/gnss-analysis/assignees{/user}","branches_url":"https://api.github.com/repos/swift-nav/gnss-analysis/branches{/branch}","tags_url":"https://api.github.com/repos/swift-nav/gnss-analysis/tags","blobs_url":"https://api.github.com/repos/swift-nav/gnss-analysis/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/swift-nav/gnss-analysis/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/swift-nav/gnss-analysis/git/refs{/sha}","trees_url":"https://api.github.com/repos/swift-nav/gnss-analysis/git/trees{/sha}","statuses_url":"https://api.github.com/repos/swift-nav/gnss-analysis/statuses/{sha}","languages_url":"https://api.github.com/repos/swift-nav/gnss-analysis/languages","stargazers_url":"https://api.github.com/repos/swift-nav/gnss-analysis/stargazers","contributors_url":"https://api.github.com/repos/swift-nav/gnss-analysis/contributors","subscribers_url":"https://api.github.com/repos/swift-nav/gnss-analysis/subscribers","subscription_url":"https://api.github.com/repos/swift-nav/gnss-analysis/subscription","commits_url":"https://api.github.com/repos/swift-nav/gnss-analysis/commits{/sha}","git_commits_url":"https://api.github.com/repos/swift-nav/gnss-analysis/git/commits{/sha}","comments_url":"https://api.github.com/repos/swift-nav/gnss-analysis/comments{/number}","issue_comment_url":"https://api.github.com/repos/swift-nav/gnss-analysis/issues/comments{/number}","contents_url":"https://api.github.com/repos/swift-nav/gnss-analysis/contents/{+path}","compare_url":"https://api.github.com/repos/swift-nav/gnss-analysis/compare/{base}...{head}","merges_url":"https://api.github.com/repos/swift-nav/gnss-analysis/merges","archive_url":"https://api.github.com/repos/swift-nav/gnss-analysis/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/swift-nav/gnss-analysis/downloads","issues_url":"https://api.github.com/repos/swift-nav/gnss-analysis/issues{/number}","pulls_url":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls{/number}","milestones_url":"https://api.github.com/repos/swift-nav/gnss-analysis/milestones{/number}","notifications_url":"https://api.github.com/repos/swift-nav/gnss-analysis/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/swift-nav/gnss-analysis/labels{/name}","releases_url":"https://api.github.com/repos/swift-nav/gnss-analysis/releases{/id}","created_at":"2014-04-22T21:49:48Z","updated_at":"2015-05-11T02:44:45Z","pushed_at":"2015-11-18T23:24:41Z","git_url":"git://github.com/swift-nav/gnss-analysis.git","ssh_url":"git@github.com:swift-nav/gnss-analysis.git","clone_url":"https://github.com/swift-nav/gnss-analysis.git","svn_url":"https://github.com/swift-nav/gnss-analysis","homepage":"","size":15242,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":9,"mirror_url":null,"open_issues_count":10,"forks":9,"open_issues":10,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/70"},"html":{"href":"https://github.com/swift-nav/gnss-analysis/pull/70"},"issue":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/issues/70"},"comments":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/issues/70/comments"},"review_comments":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/70/comments"},"review_comment":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/pulls/70/commits"},"statuses":{"href":"https://api.github.com/repos/swift-nav/gnss-analysis/statuses/6ce8aa4b065f71d415ae8f2412fbc9fb6348c8d2"}}}}