C# WebBrowser Session to WebRequest
<p>I'm trying to write a program to scrape urls from google, and when asked for a captcha, will open a form to allow the user to type the captcha in and let the program continue. The program works fine till the captcha. The form will open and allow the user to type in the captcha, and the webbrowser will load the next page fine, but the session will not be carried over to the webrequest, thus causing a loop of opening the webbrowser form asking the user to type in the captcha. I have tried copying over the cookies from the webbrowser to the webrequest cookie container, but no avail.</p>

<pre><code>            foreach (string cookie in f2.webForm.Document.Cookie.Split(';'))
            {
                string name = cookie.Split('=')[0];
                string value = cookie.Substring(name.Length + 1);
                string path = "/";
                string domain = "ipv4.google.com";
                //webRequest.CookieContainer.Add(new Cookie(name.Trim(), value.Trim(), path, domain));
                cookieJar.Add(new Cookie(name.Trim(), value.Trim(), path, domain));
            }
</code></pre>

<p>Here is the full code. Please keep in mind it's a little roughly written, so don't judge :P</p>

<pre><code>    CookieContainer cookieJar = new CookieContainer();
    for (int i = 0; i &lt;= 30; i += 10)
    {
        string url = "https://www.google.com/search?newwindow=1&amp;q=inurl:test.php" + "&amp;start=" + i;
        HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(url);
        webRequest.CookieContainer = cookieJar;
        Thread.Sleep(1000);
        try
        {
            webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246";
            //webRequest.CookieContainer = new CookieContainer();
            webRequest.ProtocolVersion = HttpVersion.Version11;
            webRequest.Method = "GET";
            webRequest.KeepAlive = false;
            webRequest.ContentType = "text/html";
            webRequest.Timeout = 20000;
            //webRequest.UseDefaultCredentials = true;
            Stream objStream = webRequest.GetResponse().GetResponseStream();
            StreamReader streamReader = new StreamReader(objStream);
            String sLine = "";
            List&lt;string&gt; lLines = new List&lt;string&gt;();
            List&lt;string&gt; lUrls = new List&lt;string&gt;();
            string[] findhttp;
            int endIndex = 0;
            Thread.Sleep(1000);
            HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
            boxUrl.AppendText("test: " + webResponse.StatusCode + "\n");

            // Get Google's web search and store each line in "lUrls" List
            while (sLine != null)
            {
                boxDorks.AppendText(sLine);
                lLines.Add(sLine);
                sLine = streamReader.ReadLine();
            }

            // Lets loop through and get all the URLs
            foreach (string s in lLines)
            {
                // Find the index of href="http
                findhttp = s.Split(new string[] { "href=\"http" }, StringSplitOptions.None);

                // Parse URL
                foreach (string find in findhttp)
                {
                    if (s.IndexOf("href=\"http") &gt; 0)
                    {
                        endIndex = find.IndexOf("\" onmousedown"); // Find position of quote

                        if (endIndex &gt; 0 &amp;&amp; find.IndexOf("webcache.googleusercontent.com") &lt; 0 &amp;&amp;
                                            find.IndexOf("support.google.com") &lt; 0 &amp;&amp;
                                            find.IndexOf("robots.txt") &lt; 0 &amp;&amp;
                                            find.IndexOf("translate.google.com") &lt; 0) // we don't want these!
                        {
                            lUrls.Add("http" + find.Substring(0, endIndex));
                        }
                    }
                }
            }

            // Output URLs
            foreach (string s in lUrls)
            {
                boxUrl.AppendText("test: " + s + "\n");
            }
        }
        catch (WebException we)
        {
            boxUrl.AppendText("exception: " + we);
            //using (var sr = new StreamReader(we.Response.GetResponseStream()))
            // {
            //var html = sr.ReadToEnd();
              //}
            // Open form to show google captcha
            Form2 f2 = new Form2(we.Response.ResponseUri.ToString());//workaround to get webform.Navigate to work properly
            f2.ShowDialog();

            // Copy cookies from webbrowser to webrequest cookies
            foreach (string cookie in f2.webForm.Document.Cookie.Split(';'))
            {
                string name = cookie.Split('=')[0];
                string value = cookie.Substring(name.Length + 1);
                string path = "/";
                string domain = "ipv4.google.com";
                //webRequest.CookieContainer.Add(new Cookie(name.Trim(), value.Trim(), path, domain));
                cookieJar.Add(new Cookie(name.Trim(), value.Trim(), path, domain));
            }
</code></pre>

<p>Thank you in advance!</p>