{"action":"created","comment":{"url":"https://api.github.com/repos/gonum/optimize/pulls/comments/29912326","id":29912326,"diff_hunk":"@@ -0,0 +1,143 @@\n+// Copyright ©2015 The gonum Authors. All rights reserved.\n+// Use of this source code is governed by a BSD-style\n+// license that can be found in the LICENSE file.\n+\n+package optimize\n+\n+import (\n+\t\"math\"\n+\n+\t\"github.com/gonum/floats\"\n+\t\"github.com/gonum/matrix/mat64\"\n+)\n+\n+// Newton implements a modified Newton's method for Hessian-based unconstrained\n+// minimization. It applies regularization when the Hessian is not positive\n+// definite, and uses line-searches so it can converge to a local minimum from\n+// any starting point.\n+//\n+// Newton iteratively forms a quadratic model to the objective function f and\n+// tries to minimize this approximate model. It generates a sequence of\n+// locations x_k by means of\n+//  solve H_k d_k = -∇f_k for d_k,\n+//  x_{k+1} = x_k + α_k d_k,\n+// where H_k is the Hessian matrix of f at x_k and α_k is a step size found by\n+// a line search.\n+//\n+// Away from a minimizer H_k may not be positive definite and d_k may not be a\n+// descent direction. Newton implements a Hessian modification strategy that\n+// tries to add successively larger multiples of identity to H_k until it\n+// becomes positive definite. Note that the repeated trial factorization of the\n+// modified Hessian involved in this process can be computationally expensive.\n+//\n+// If the Hessian matrix cannot be formed explicitly or if the computational\n+// cost of its factorization is prohibitive, BFGS or L-BFGS quasi-Newton method\n+// can be used instead.\n+type Newton struct {\n+\t// LinesearchMethod is a method used for selecting suitable steps along the\n+\t// descent direction d. Steps should satisfy the Wolfe, Goldstein or Armijo\n+\t// conditions. If LinesearchMethod == nil, an appropriate default is\n+\t// chosen.\n+\tLinesearchMethod LinesearchMethod\n+\t// Increase is a factor by which a scalar tau is successively increased so\n+\t// that (H + tau*I) is positive definite. Larger values reduce the number\n+\t// of trial Hessian factorizations, but also reduce the second-order\n+\t// information in H.\n+\t// Increase must be greater than 1. If Increase is 0, it is defaulted to 5.\n+\tIncrease float64\n+\n+\tlinesearch *Linesearch\n+\n+\thess *mat64.SymDense\n+\tchol *mat64.TriDense\n+\ttau  float64\n+}\n+\n+func (n *Newton) Init(loc *Location, f *FunctionInfo, xNext []float64) (EvaluationType, IterationType, error) {\n+\tif n.Increase == 0 {\n+\t\tn.Increase = 5\n+\t}\n+\tif n.Increase <= 1 {\n+\t\tpanic(\"optimize: Newton.Increase must be greater than 1\")\n+\t}\n+\tif n.LinesearchMethod == nil {\n+\t\tn.LinesearchMethod = &Bisection{}\n+\t}\n+\tif n.linesearch == nil {\n+\t\tn.linesearch = &Linesearch{}\n+\t}\n+\tn.linesearch.Method = n.LinesearchMethod\n+\tn.linesearch.NextDirectioner = n\n+\n+\treturn n.linesearch.Init(loc, f, xNext)\n+}\n+\n+func (n *Newton) Iterate(loc *Location, xNext []float64) (EvaluationType, IterationType, error) {\n+\treturn n.linesearch.Iterate(loc, xNext)\n+}\n+\n+func (n *Newton) InitDirection(loc *Location, dir []float64) (stepSize float64) {\n+\tdim := len(loc.X)\n+\tn.chol = resizeTriDense(n.chol, dim)\n+\tn.hess = resizeSymDense(n.hess, dim)\n+\tn.tau = 0\n+\tn.computeNextDir(loc, dir)\n+\treturn 1\n+}\n+\n+func (n *Newton) NextDirection(loc *Location, dir []float64) (stepSize float64) {\n+\tn.computeNextDir(loc, dir)\n+\treturn 1\n+}\n+\n+func (n *Newton) computeNextDir(loc *Location, dir []float64) {\n+\tdim := len(loc.X)\n+\tn.hess.CopySym(loc.Hessian)\n+\n+\t// Find the smallest diagonal element of H.\n+\tminA := n.hess.At(0, 0)\n+\tfor i := 1; i < dim; i++ {\n+\t\ta := n.hess.At(i, i)\n+\t\tif a < minA {\n+\t\t\tminA = a\n+\t\t}\n+\t}\n+\tif minA > 0 {\n+\t\t// All diagonal elements are positive, so try to factorize non-modified\n+\t\t// Hessian first.\n+\t\tn.tau = 0\n+\t} else if n.tau == 0 {\n+\t\t// Use the last tau that gave PD modified Hessian.\n+\t\tn.tau = -minA + 0.001\n+\t}\n+\n+\tfor {\n+\t\tif n.tau != 0 {\n+\t\t\t// Add a multiple of identity to the Hessian.\n+\t\t\tfor i := 0; i < dim; i++ {\n+\t\t\t\tn.hess.SetSym(i, i, loc.Hessian.At(i, i)+n.tau)\n+\t\t\t}\n+\t\t}\n+\t\t// Try to apply the Cholesky decomposition.\n+\t\tpd := n.chol.Cholesky(n.hess, false)\n+\t\tif pd {\n+\t\t\tbreak\n+\t\t}\n+\t\t// Modified Hessian is not PD, so increase tau.\n+\t\tn.tau = math.Max(n.Increase*n.tau, 0.001)\n+\t}\n+\td := mat64.NewDense(dim, 1, dir)\n+\t// Store the solution in d's backing array, dir.","path":"newton.go","position":null,"original_position":130,"commit_id":"5fc43f69eb014435e17b97e732ad3c05dccf797a","original_commit_id":"9a2b82af4b4fa46576cdcb31270b9b57f176569b","user":{"login":"vladimir-ch","id":1837502,"avatar_url":"https://avatars.githubusercontent.com/u/1837502?v=3","gravatar_id":"","url":"https://api.github.com/users/vladimir-ch","html_url":"https://github.com/vladimir-ch","followers_url":"https://api.github.com/users/vladimir-ch/followers","following_url":"https://api.github.com/users/vladimir-ch/following{/other_user}","gists_url":"https://api.github.com/users/vladimir-ch/gists{/gist_id}","starred_url":"https://api.github.com/users/vladimir-ch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vladimir-ch/subscriptions","organizations_url":"https://api.github.com/users/vladimir-ch/orgs","repos_url":"https://api.github.com/users/vladimir-ch/repos","events_url":"https://api.github.com/users/vladimir-ch/events{/privacy}","received_events_url":"https://api.github.com/users/vladimir-ch/received_events","type":"User","site_admin":false},"body":"Done","created_at":"2015-05-08T01:53:49Z","updated_at":"2015-05-08T01:53:49Z","html_url":"https://github.com/gonum/optimize/pull/99#discussion_r29912326","pull_request_url":"https://api.github.com/repos/gonum/optimize/pulls/99","_links":{"self":{"href":"https://api.github.com/repos/gonum/optimize/pulls/comments/29912326"},"html":{"href":"https://github.com/gonum/optimize/pull/99#discussion_r29912326"},"pull_request":{"href":"https://api.github.com/repos/gonum/optimize/pulls/99"}}},"pull_request":{"url":"https://api.github.com/repos/gonum/optimize/pulls/99","id":33639933,"html_url":"https://github.com/gonum/optimize/pull/99","diff_url":"https://github.com/gonum/optimize/pull/99.diff","patch_url":"https://github.com/gonum/optimize/pull/99.patch","issue_url":"https://api.github.com/repos/gonum/optimize/issues/99","number":99,"state":"open","locked":false,"title":"Add Newton method","user":{"login":"vladimir-ch","id":1837502,"avatar_url":"https://avatars.githubusercontent.com/u/1837502?v=3","gravatar_id":"","url":"https://api.github.com/users/vladimir-ch","html_url":"https://github.com/vladimir-ch","followers_url":"https://api.github.com/users/vladimir-ch/followers","following_url":"https://api.github.com/users/vladimir-ch/following{/other_user}","gists_url":"https://api.github.com/users/vladimir-ch/gists{/gist_id}","starred_url":"https://api.github.com/users/vladimir-ch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vladimir-ch/subscriptions","organizations_url":"https://api.github.com/users/vladimir-ch/orgs","repos_url":"https://api.github.com/users/vladimir-ch/repos","events_url":"https://api.github.com/users/vladimir-ch/events{/privacy}","received_events_url":"https://api.github.com/users/vladimir-ch/received_events","type":"User","site_admin":false},"body":"Has the time really come for Newton?\r\n\r\nAn implementation of a pure, locally convergent Newton method without linesearch and of a modified, globally convergent Newton method with a linesearch.\r\n\r\nThe pure Newton is rather simple now that mat64 has symmetric matrices and Cholesky decomposition.\r\n\r\nThe modification in ModifiedNewton is a simple one, just add multiples of identity to the Hessian until it becomes positive definite. I did not venture into the modified Cholesky method at the moment but I can if there is a desire to have it.\r\n\r\nI added three functions for resizing/reusing matrices in a similar way we already resize/reuse slices. Experts on matrix package @kortschak @btracey might have a better suggestion.\r\n\r\nNo tests at the moment, first we need to add Hess() method to functions, then modify the testing itself to accomodate Hessian-based methods.\r\n\r\nPTAL, @btracey ","created_at":"2015-04-20T07:03:07Z","updated_at":"2015-05-08T01:53:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"edd0f5c652616049fc0c0e73b6dfddde45028948","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/gonum/optimize/pulls/99/commits","review_comments_url":"https://api.github.com/repos/gonum/optimize/pulls/99/comments","review_comment_url":"https://api.github.com/repos/gonum/optimize/pulls/comments{/number}","comments_url":"https://api.github.com/repos/gonum/optimize/issues/99/comments","statuses_url":"https://api.github.com/repos/gonum/optimize/statuses/5fc43f69eb014435e17b97e732ad3c05dccf797a","head":{"label":"gonum:add-newton","ref":"add-newton","sha":"5fc43f69eb014435e17b97e732ad3c05dccf797a","user":{"login":"gonum","id":3771091,"avatar_url":"https://avatars.githubusercontent.com/u/3771091?v=3","gravatar_id":"","url":"https://api.github.com/users/gonum","html_url":"https://github.com/gonum","followers_url":"https://api.github.com/users/gonum/followers","following_url":"https://api.github.com/users/gonum/following{/other_user}","gists_url":"https://api.github.com/users/gonum/gists{/gist_id}","starred_url":"https://api.github.com/users/gonum/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gonum/subscriptions","organizations_url":"https://api.github.com/users/gonum/orgs","repos_url":"https://api.github.com/users/gonum/repos","events_url":"https://api.github.com/users/gonum/events{/privacy}","received_events_url":"https://api.github.com/users/gonum/received_events","type":"Organization","site_admin":false},"repo":{"id":24917465,"name":"optimize","full_name":"gonum/optimize","owner":{"login":"gonum","id":3771091,"avatar_url":"https://avatars.githubusercontent.com/u/3771091?v=3","gravatar_id":"","url":"https://api.github.com/users/gonum","html_url":"https://github.com/gonum","followers_url":"https://api.github.com/users/gonum/followers","following_url":"https://api.github.com/users/gonum/following{/other_user}","gists_url":"https://api.github.com/users/gonum/gists{/gist_id}","starred_url":"https://api.github.com/users/gonum/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gonum/subscriptions","organizations_url":"https://api.github.com/users/gonum/orgs","repos_url":"https://api.github.com/users/gonum/repos","events_url":"https://api.github.com/users/gonum/events{/privacy}","received_events_url":"https://api.github.com/users/gonum/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/gonum/optimize","description":"Package for finding the optimum value of functions.","fork":false,"url":"https://api.github.com/repos/gonum/optimize","forks_url":"https://api.github.com/repos/gonum/optimize/forks","keys_url":"https://api.github.com/repos/gonum/optimize/keys{/key_id}","collaborators_url":"https://api.github.com/repos/gonum/optimize/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/gonum/optimize/teams","hooks_url":"https://api.github.com/repos/gonum/optimize/hooks","issue_events_url":"https://api.github.com/repos/gonum/optimize/issues/events{/number}","events_url":"https://api.github.com/repos/gonum/optimize/events","assignees_url":"https://api.github.com/repos/gonum/optimize/assignees{/user}","branches_url":"https://api.github.com/repos/gonum/optimize/branches{/branch}","tags_url":"https://api.github.com/repos/gonum/optimize/tags","blobs_url":"https://api.github.com/repos/gonum/optimize/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/gonum/optimize/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/gonum/optimize/git/refs{/sha}","trees_url":"https://api.github.com/repos/gonum/optimize/git/trees{/sha}","statuses_url":"https://api.github.com/repos/gonum/optimize/statuses/{sha}","languages_url":"https://api.github.com/repos/gonum/optimize/languages","stargazers_url":"https://api.github.com/repos/gonum/optimize/stargazers","contributors_url":"https://api.github.com/repos/gonum/optimize/contributors","subscribers_url":"https://api.github.com/repos/gonum/optimize/subscribers","subscription_url":"https://api.github.com/repos/gonum/optimize/subscription","commits_url":"https://api.github.com/repos/gonum/optimize/commits{/sha}","git_commits_url":"https://api.github.com/repos/gonum/optimize/git/commits{/sha}","comments_url":"https://api.github.com/repos/gonum/optimize/comments{/number}","issue_comment_url":"https://api.github.com/repos/gonum/optimize/issues/comments{/number}","contents_url":"https://api.github.com/repos/gonum/optimize/contents/{+path}","compare_url":"https://api.github.com/repos/gonum/optimize/compare/{base}...{head}","merges_url":"https://api.github.com/repos/gonum/optimize/merges","archive_url":"https://api.github.com/repos/gonum/optimize/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/gonum/optimize/downloads","issues_url":"https://api.github.com/repos/gonum/optimize/issues{/number}","pulls_url":"https://api.github.com/repos/gonum/optimize/pulls{/number}","milestones_url":"https://api.github.com/repos/gonum/optimize/milestones{/number}","notifications_url":"https://api.github.com/repos/gonum/optimize/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/gonum/optimize/labels{/name}","releases_url":"https://api.github.com/repos/gonum/optimize/releases{/id}","created_at":"2014-10-08T00:14:50Z","updated_at":"2015-05-08T00:32:33Z","pushed_at":"2015-05-08T01:47:36Z","git_url":"git://github.com/gonum/optimize.git","ssh_url":"git@github.com:gonum/optimize.git","clone_url":"https://github.com/gonum/optimize.git","svn_url":"https://github.com/gonum/optimize","homepage":null,"size":2517,"stargazers_count":19,"watchers_count":19,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":16,"forks":1,"open_issues":16,"watchers":19,"default_branch":"master"}},"base":{"label":"gonum:master","ref":"master","sha":"8ec0af7cbeb6e91aa9fdad2acf50a2aaf61a8c04","user":{"login":"gonum","id":3771091,"avatar_url":"https://avatars.githubusercontent.com/u/3771091?v=3","gravatar_id":"","url":"https://api.github.com/users/gonum","html_url":"https://github.com/gonum","followers_url":"https://api.github.com/users/gonum/followers","following_url":"https://api.github.com/users/gonum/following{/other_user}","gists_url":"https://api.github.com/users/gonum/gists{/gist_id}","starred_url":"https://api.github.com/users/gonum/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gonum/subscriptions","organizations_url":"https://api.github.com/users/gonum/orgs","repos_url":"https://api.github.com/users/gonum/repos","events_url":"https://api.github.com/users/gonum/events{/privacy}","received_events_url":"https://api.github.com/users/gonum/received_events","type":"Organization","site_admin":false},"repo":{"id":24917465,"name":"optimize","full_name":"gonum/optimize","owner":{"login":"gonum","id":3771091,"avatar_url":"https://avatars.githubusercontent.com/u/3771091?v=3","gravatar_id":"","url":"https://api.github.com/users/gonum","html_url":"https://github.com/gonum","followers_url":"https://api.github.com/users/gonum/followers","following_url":"https://api.github.com/users/gonum/following{/other_user}","gists_url":"https://api.github.com/users/gonum/gists{/gist_id}","starred_url":"https://api.github.com/users/gonum/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gonum/subscriptions","organizations_url":"https://api.github.com/users/gonum/orgs","repos_url":"https://api.github.com/users/gonum/repos","events_url":"https://api.github.com/users/gonum/events{/privacy}","received_events_url":"https://api.github.com/users/gonum/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/gonum/optimize","description":"Package for finding the optimum value of functions.","fork":false,"url":"https://api.github.com/repos/gonum/optimize","forks_url":"https://api.github.com/repos/gonum/optimize/forks","keys_url":"https://api.github.com/repos/gonum/optimize/keys{/key_id}","collaborators_url":"https://api.github.com/repos/gonum/optimize/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/gonum/optimize/teams","hooks_url":"https://api.github.com/repos/gonum/optimize/hooks","issue_events_url":"https://api.github.com/repos/gonum/optimize/issues/events{/number}","events_url":"https://api.github.com/repos/gonum/optimize/events","assignees_url":"https://api.github.com/repos/gonum/optimize/assignees{/user}","branches_url":"https://api.github.com/repos/gonum/optimize/branches{/branch}","tags_url":"https://api.github.com/repos/gonum/optimize/tags","blobs_url":"https://api.github.com/repos/gonum/optimize/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/gonum/optimize/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/gonum/optimize/git/refs{/sha}","trees_url":"https://api.github.com/repos/gonum/optimize/git/trees{/sha}","statuses_url":"https://api.github.com/repos/gonum/optimize/statuses/{sha}","languages_url":"https://api.github.com/repos/gonum/optimize/languages","stargazers_url":"https://api.github.com/repos/gonum/optimize/stargazers","contributors_url":"https://api.github.com/repos/gonum/optimize/contributors","subscribers_url":"https://api.github.com/repos/gonum/optimize/subscribers","subscription_url":"https://api.github.com/repos/gonum/optimize/subscription","commits_url":"https://api.github.com/repos/gonum/optimize/commits{/sha}","git_commits_url":"https://api.github.com/repos/gonum/optimize/git/commits{/sha}","comments_url":"https://api.github.com/repos/gonum/optimize/comments{/number}","issue_comment_url":"https://api.github.com/repos/gonum/optimize/issues/comments{/number}","contents_url":"https://api.github.com/repos/gonum/optimize/contents/{+path}","compare_url":"https://api.github.com/repos/gonum/optimize/compare/{base}...{head}","merges_url":"https://api.github.com/repos/gonum/optimize/merges","archive_url":"https://api.github.com/repos/gonum/optimize/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/gonum/optimize/downloads","issues_url":"https://api.github.com/repos/gonum/optimize/issues{/number}","pulls_url":"https://api.github.com/repos/gonum/optimize/pulls{/number}","milestones_url":"https://api.github.com/repos/gonum/optimize/milestones{/number}","notifications_url":"https://api.github.com/repos/gonum/optimize/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/gonum/optimize/labels{/name}","releases_url":"https://api.github.com/repos/gonum/optimize/releases{/id}","created_at":"2014-10-08T00:14:50Z","updated_at":"2015-05-08T00:32:33Z","pushed_at":"2015-05-08T01:47:36Z","git_url":"git://github.com/gonum/optimize.git","ssh_url":"git@github.com:gonum/optimize.git","clone_url":"https://github.com/gonum/optimize.git","svn_url":"https://github.com/gonum/optimize","homepage":null,"size":2517,"stargazers_count":19,"watchers_count":19,"language":"Go","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":16,"forks":1,"open_issues":16,"watchers":19,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/gonum/optimize/pulls/99"},"html":{"href":"https://github.com/gonum/optimize/pull/99"},"issue":{"href":"https://api.github.com/repos/gonum/optimize/issues/99"},"comments":{"href":"https://api.github.com/repos/gonum/optimize/issues/99/comments"},"review_comments":{"href":"https://api.github.com/repos/gonum/optimize/pulls/99/comments"},"review_comment":{"href":"https://api.github.com/repos/gonum/optimize/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/gonum/optimize/pulls/99/commits"},"statuses":{"href":"https://api.github.com/repos/gonum/optimize/statuses/5fc43f69eb014435e17b97e732ad3c05dccf797a"}}}}