{"action":"created","comment":{"url":"https://api.github.com/repos/mapsme/omim/pulls/comments/46280490","id":46280490,"diff_hunk":"@@ -0,0 +1,233 @@\n+#include \"search/v2/geocoder.hpp\"\n+\n+#include \"search/retrieval.hpp\"\n+#include \"search/v2/features_layer_path_finder.hpp\"\n+\n+#include \"indexer/feature_decl.hpp\"\n+#include \"indexer/index.hpp\"\n+#include \"indexer/search_delimiters.hpp\"\n+#include \"indexer/search_string_utils.hpp\"\n+\n+#include \"coding/multilang_utf8_string.hpp\"\n+\n+#include \"platform/preferred_languages.hpp\"\n+\n+#include \"base/assert.hpp\"\n+#include \"base/logging.hpp\"\n+#include \"base/macros.hpp\"\n+#include \"base/scope_guard.hpp\"\n+#include \"base/stl_add.hpp\"\n+\n+#include \"std/algorithm.hpp\"\n+#include \"std/iterator.hpp\"\n+\n+namespace search\n+{\n+namespace v2\n+{\n+Geocoder::Geocoder(Index & index)\n+  : m_index(index)\n+  , m_numTokens(0)\n+  , m_model(SearchModel::Instance())\n+  , m_value(nullptr)\n+  , m_results(nullptr)\n+{\n+}\n+\n+Geocoder::~Geocoder() {}\n+\n+void Geocoder::SetSearchQueryParams(SearchQueryParams const & params)\n+{\n+  m_params = params;\n+  m_retrievalParams = params;\n+\n+  m_numTokens = m_params.m_tokens.size();\n+  if (!m_params.m_prefixTokens.empty())\n+    ++m_numTokens;\n+}\n+\n+void Geocoder::Go(vector<FeatureID> & results)\n+{\n+  m_results = &results;\n+\n+  try\n+  {\n+    vector<shared_ptr<MwmInfo>> mwmsInfo;\n+    m_index.GetMwmsInfo(mwmsInfo);\n+\n+    // Iterate through all alive mwms and perform geocoding.\n+    for (auto const & info : mwmsInfo)\n+    {\n+      auto handle = m_index.GetMwmHandleById(MwmSet::MwmId(info));\n+      if (!handle.IsAlive())\n+        continue;\n+\n+      m_value = handle.GetValue<MwmValue>();\n+      if (!m_value || !m_value->m_cont.IsExist(SEARCH_INDEX_FILE_TAG))\n+        continue;\n+\n+      m_mwmId = handle.GetId();\n+\n+      MY_SCOPE_GUARD(cleanup, [&]()\n+                     {\n+        m_finder.reset();\n+        m_loader.reset();\n+        m_cache.clear();\n+      });\n+\n+      m_loader.reset(new Index::FeaturesLoaderGuard(m_index, m_mwmId));\n+      m_finder.reset(new FeaturesLayerPathFinder(m_loader->GetFeaturesVector()));\n+      m_cache.clear();\n+\n+      DoGeocoding(0 /* curToken */);\n+    }\n+  }\n+  catch (CancelException & e)\n+  {\n+  }\n+}\n+\n+void Geocoder::PrepareParams(size_t from, size_t to)\n+{\n+  ASSERT_LESS(from, to, ());\n+  ASSERT_LESS_OR_EQUAL(to, m_numTokens, ());\n+\n+  m_retrievalParams.m_tokens.clear();\n+  m_retrievalParams.m_prefixTokens.clear();\n+\n+  for (size_t i = from; i < to; ++i)\n+  {\n+    if (i < m_params.m_tokens.size())\n+      m_retrievalParams.m_tokens.push_back(m_params.m_tokens[i]);\n+    else\n+      m_retrievalParams.m_prefixTokens = m_params.m_prefixTokens;\n+  }\n+}\n+\n+void Geocoder::DoGeocoding(size_t curToken)\n+{\n+  if (curToken == m_numTokens)\n+  {\n+    // All tokens were consumed, intersect layers, emit features.\n+    IntersectLayers();\n+    return;\n+  }\n+\n+  m_layers.emplace_back();\n+  MY_SCOPE_GUARD(cleanupGuard, bind(&vector<FeaturesLayer>::pop_back, &m_layers));\n+\n+  // Try to consume first n tokens starting at |curToken|.\n+  for (size_t n = 1; curToken + n <= m_numTokens; ++n)\n+  {\n+    PrepareParams(curToken, curToken + n);\n+    {\n+      auto & layer = m_layers.back();\n+      layer.Clear();\n+      layer.m_startToken = curToken;\n+      layer.m_endToken = curToken + n;\n+    }\n+\n+    auto features = RetrieveAddressFeatures(curToken, curToken + n);\n+    if (!features || features->PopCount() == 0)\n+      continue;\n+\n+    vector<uint32_t> clusters[SearchModel::SEARCH_TYPE_COUNT];\n+    auto clusterize = [&](uint64_t featureId)\n+    {\n+      FeatureType feature;\n+      m_loader->GetFeatureByIndex(featureId, feature);\n+      feature.ParseTypes();\n+      SearchModel::SearchType searchType = m_model.GetSearchType(feature);\n+      if (searchType != SearchModel::SEARCH_TYPE_COUNT)\n+        clusters[searchType].push_back(featureId);\n+    };\n+\n+    coding::CompressedBitVectorEnumerator::ForEach(*features, clusterize);\n+\n+    bool const looksLikeHouseNumber = LooksLikeHouseNumber(curToken, curToken + n);\n+\n+    for (size_t i = 0; i != SearchModel::SEARCH_TYPE_COUNT; ++i)\n+    {\n+      if (i == SearchModel::SEARCH_TYPE_BUILDING)\n+      {\n+        if (clusters[i].empty() && !looksLikeHouseNumber)\n+          continue;\n+      }\n+      else if (clusters[i].empty())\n+      {\n+        continue;\n+      }\n+\n+      // ATTENTION: DO NOT USE layer after recursive calls to\n+      // DoGeocoding().  This may lead to use-after-free.\n+      auto & layer = m_layers.back();\n+\n+      layer.m_features.swap(clusters[i]);\n+      layer.m_type = static_cast<SearchModel::SearchType>(i);\n+      if (IsLayerSequenceSane())\n+        DoGeocoding(curToken + n);\n+    }\n+  }\n+}\n+\n+coding::CompressedBitVector * Geocoder::RetrieveAddressFeatures(size_t curToken, size_t endToken)\n+{\n+  uint64_t const key = (static_cast<uint64_t>(curToken) << 32) | static_cast<uint64_t>(endToken);\n+  if (m_cache.find(key) == m_cache.end())\n+  {\n+    m_cache[key] =\n+        Retrieval::RetrieveAddressFeatures(m_value, *this /* cancellable */, m_retrievalParams);\n+  }\n+  return m_cache[key].get();\n+}\n+\n+bool Geocoder::IsLayerSequenceSane() const","path":"search/v2/geocoder.cpp","position":188,"original_position":184,"commit_id":"c3dfc07c788875382652a4ea00266a1c3bb016c7","original_commit_id":"658f3617e64fb1e619dbc825ee44fd5758238d83","user":{"login":"vng","id":175612,"avatar_url":"https://avatars.githubusercontent.com/u/175612?v=3","gravatar_id":"","url":"https://api.github.com/users/vng","html_url":"https://github.com/vng","followers_url":"https://api.github.com/users/vng/followers","following_url":"https://api.github.com/users/vng/following{/other_user}","gists_url":"https://api.github.com/users/vng/gists{/gist_id}","starred_url":"https://api.github.com/users/vng/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vng/subscriptions","organizations_url":"https://api.github.com/users/vng/orgs","repos_url":"https://api.github.com/users/vng/repos","events_url":"https://api.github.com/users/vng/events{/privacy}","received_events_url":"https://api.github.com/users/vng/received_events","type":"User","site_admin":false},"body":"Не совсем понятно что значит \"sane\". Может назвать что она реально делает - по коду проверяет что m_layers не пересекаются по типам ...","created_at":"2015-12-01T14:01:48Z","updated_at":"2015-12-01T14:01:48Z","html_url":"https://github.com/mapsme/omim/pull/716#discussion_r46280490","pull_request_url":"https://api.github.com/repos/mapsme/omim/pulls/716","_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments/46280490"},"html":{"href":"https://github.com/mapsme/omim/pull/716#discussion_r46280490"},"pull_request":{"href":"https://api.github.com/repos/mapsme/omim/pulls/716"}}},"pull_request":{"url":"https://api.github.com/repos/mapsme/omim/pulls/716","id":51808237,"html_url":"https://github.com/mapsme/omim/pull/716","diff_url":"https://github.com/mapsme/omim/pull/716.diff","patch_url":"https://github.com/mapsme/omim/pull/716.patch","issue_url":"https://api.github.com/repos/mapsme/omim/issues/716","number":716,"state":"open","locked":false,"title":"[search] Skeleton of a SearchQueryV2 (Geocoder).","user":{"login":"ygorshenin","id":454567,"avatar_url":"https://avatars.githubusercontent.com/u/454567?v=3","gravatar_id":"","url":"https://api.github.com/users/ygorshenin","html_url":"https://github.com/ygorshenin","followers_url":"https://api.github.com/users/ygorshenin/followers","following_url":"https://api.github.com/users/ygorshenin/following{/other_user}","gists_url":"https://api.github.com/users/ygorshenin/gists{/gist_id}","starred_url":"https://api.github.com/users/ygorshenin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ygorshenin/subscriptions","organizations_url":"https://api.github.com/users/ygorshenin/orgs","repos_url":"https://api.github.com/users/ygorshenin/repos","events_url":"https://api.github.com/users/ygorshenin/events{/privacy}","received_events_url":"https://api.github.com/users/ygorshenin/received_events","type":"User","site_admin":false},"body":"@vng, @mpimenov PTAL","created_at":"2015-11-25T15:28:56Z","updated_at":"2015-12-01T14:01:48Z","closed_at":null,"merged_at":null,"merge_commit_sha":"56fd4328e2d33f1ad80d3a3330f7295740cc88bc","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mapsme/omim/pulls/716/commits","review_comments_url":"https://api.github.com/repos/mapsme/omim/pulls/716/comments","review_comment_url":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mapsme/omim/issues/716/comments","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/c3dfc07c788875382652a4ea00266a1c3bb016c7","head":{"label":"ygorshenin:search-query-v2","ref":"search-query-v2","sha":"c3dfc07c788875382652a4ea00266a1c3bb016c7","user":{"login":"ygorshenin","id":454567,"avatar_url":"https://avatars.githubusercontent.com/u/454567?v=3","gravatar_id":"","url":"https://api.github.com/users/ygorshenin","html_url":"https://github.com/ygorshenin","followers_url":"https://api.github.com/users/ygorshenin/followers","following_url":"https://api.github.com/users/ygorshenin/following{/other_user}","gists_url":"https://api.github.com/users/ygorshenin/gists{/gist_id}","starred_url":"https://api.github.com/users/ygorshenin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ygorshenin/subscriptions","organizations_url":"https://api.github.com/users/ygorshenin/orgs","repos_url":"https://api.github.com/users/ygorshenin/repos","events_url":"https://api.github.com/users/ygorshenin/events{/privacy}","received_events_url":"https://api.github.com/users/ygorshenin/received_events","type":"User","site_admin":false},"repo":{"id":43427948,"name":"omim","full_name":"ygorshenin/omim","owner":{"login":"ygorshenin","id":454567,"avatar_url":"https://avatars.githubusercontent.com/u/454567?v=3","gravatar_id":"","url":"https://api.github.com/users/ygorshenin","html_url":"https://github.com/ygorshenin","followers_url":"https://api.github.com/users/ygorshenin/followers","following_url":"https://api.github.com/users/ygorshenin/following{/other_user}","gists_url":"https://api.github.com/users/ygorshenin/gists{/gist_id}","starred_url":"https://api.github.com/users/ygorshenin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ygorshenin/subscriptions","organizations_url":"https://api.github.com/users/ygorshenin/orgs","repos_url":"https://api.github.com/users/ygorshenin/repos","events_url":"https://api.github.com/users/ygorshenin/events{/privacy}","received_events_url":"https://api.github.com/users/ygorshenin/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ygorshenin/omim","description":"MAPS.ME - Offline OpenStreetMap maps for iOS/Android/Mac/Linux/Windows","fork":true,"url":"https://api.github.com/repos/ygorshenin/omim","forks_url":"https://api.github.com/repos/ygorshenin/omim/forks","keys_url":"https://api.github.com/repos/ygorshenin/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ygorshenin/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ygorshenin/omim/teams","hooks_url":"https://api.github.com/repos/ygorshenin/omim/hooks","issue_events_url":"https://api.github.com/repos/ygorshenin/omim/issues/events{/number}","events_url":"https://api.github.com/repos/ygorshenin/omim/events","assignees_url":"https://api.github.com/repos/ygorshenin/omim/assignees{/user}","branches_url":"https://api.github.com/repos/ygorshenin/omim/branches{/branch}","tags_url":"https://api.github.com/repos/ygorshenin/omim/tags","blobs_url":"https://api.github.com/repos/ygorshenin/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ygorshenin/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ygorshenin/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/ygorshenin/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ygorshenin/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/ygorshenin/omim/languages","stargazers_url":"https://api.github.com/repos/ygorshenin/omim/stargazers","contributors_url":"https://api.github.com/repos/ygorshenin/omim/contributors","subscribers_url":"https://api.github.com/repos/ygorshenin/omim/subscribers","subscription_url":"https://api.github.com/repos/ygorshenin/omim/subscription","commits_url":"https://api.github.com/repos/ygorshenin/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/ygorshenin/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/ygorshenin/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/ygorshenin/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/ygorshenin/omim/contents/{+path}","compare_url":"https://api.github.com/repos/ygorshenin/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ygorshenin/omim/merges","archive_url":"https://api.github.com/repos/ygorshenin/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ygorshenin/omim/downloads","issues_url":"https://api.github.com/repos/ygorshenin/omim/issues{/number}","pulls_url":"https://api.github.com/repos/ygorshenin/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/ygorshenin/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/ygorshenin/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ygorshenin/omim/labels{/name}","releases_url":"https://api.github.com/repos/ygorshenin/omim/releases{/id}","created_at":"2015-09-30T10:56:11Z","updated_at":"2015-09-30T10:58:25Z","pushed_at":"2015-12-01T13:53:14Z","git_url":"git://github.com/ygorshenin/omim.git","ssh_url":"git@github.com:ygorshenin/omim.git","clone_url":"https://github.com/ygorshenin/omim.git","svn_url":"https://github.com/ygorshenin/omim","homepage":null,"size":1836136,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mapsme:new-search","ref":"new-search","sha":"1c6e7296cfb69f03c900cb467ab23a284a323fa1","user":{"login":"mapsme","id":14185141,"avatar_url":"https://avatars.githubusercontent.com/u/14185141?v=3","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"repo":{"id":42798033,"name":"omim","full_name":"mapsme/omim","owner":{"login":"mapsme","id":14185141,"avatar_url":"https://avatars.githubusercontent.com/u/14185141?v=3","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mapsme/omim","description":"MAPS.ME - Offline OpenStreetMap maps for iOS/Android/Mac/Linux/Windows","fork":false,"url":"https://api.github.com/repos/mapsme/omim","forks_url":"https://api.github.com/repos/mapsme/omim/forks","keys_url":"https://api.github.com/repos/mapsme/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mapsme/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mapsme/omim/teams","hooks_url":"https://api.github.com/repos/mapsme/omim/hooks","issue_events_url":"https://api.github.com/repos/mapsme/omim/issues/events{/number}","events_url":"https://api.github.com/repos/mapsme/omim/events","assignees_url":"https://api.github.com/repos/mapsme/omim/assignees{/user}","branches_url":"https://api.github.com/repos/mapsme/omim/branches{/branch}","tags_url":"https://api.github.com/repos/mapsme/omim/tags","blobs_url":"https://api.github.com/repos/mapsme/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mapsme/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mapsme/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/mapsme/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/mapsme/omim/languages","stargazers_url":"https://api.github.com/repos/mapsme/omim/stargazers","contributors_url":"https://api.github.com/repos/mapsme/omim/contributors","subscribers_url":"https://api.github.com/repos/mapsme/omim/subscribers","subscription_url":"https://api.github.com/repos/mapsme/omim/subscription","commits_url":"https://api.github.com/repos/mapsme/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/mapsme/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/mapsme/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/mapsme/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/mapsme/omim/contents/{+path}","compare_url":"https://api.github.com/repos/mapsme/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mapsme/omim/merges","archive_url":"https://api.github.com/repos/mapsme/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mapsme/omim/downloads","issues_url":"https://api.github.com/repos/mapsme/omim/issues{/number}","pulls_url":"https://api.github.com/repos/mapsme/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/mapsme/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/mapsme/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mapsme/omim/labels{/name}","releases_url":"https://api.github.com/repos/mapsme/omim/releases{/id}","created_at":"2015-09-20T02:57:13Z","updated_at":"2015-12-01T13:49:32Z","pushed_at":"2015-12-01T14:01:36Z","git_url":"git://github.com/mapsme/omim.git","ssh_url":"git@github.com:mapsme/omim.git","clone_url":"https://github.com/mapsme/omim.git","svn_url":"https://github.com/mapsme/omim","homepage":null,"size":1904209,"stargazers_count":1504,"watchers_count":1504,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":183,"mirror_url":null,"open_issues_count":67,"forks":183,"open_issues":67,"watchers":1504,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/716"},"html":{"href":"https://github.com/mapsme/omim/pull/716"},"issue":{"href":"https://api.github.com/repos/mapsme/omim/issues/716"},"comments":{"href":"https://api.github.com/repos/mapsme/omim/issues/716/comments"},"review_comments":{"href":"https://api.github.com/repos/mapsme/omim/pulls/716/comments"},"review_comment":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mapsme/omim/pulls/716/commits"},"statuses":{"href":"https://api.github.com/repos/mapsme/omim/statuses/c3dfc07c788875382652a4ea00266a1c3bb016c7"}}}}