anybody have any ideas about dealing with cookies from websites? I'm working on a page scrapper for a myspace.com account and it's looking like i may need to deal with cookies.  what i've got in mind is scraping the page contents for "setCookie" and etc, then dumping that into a text file and reading it back when necessary, but that seems pretty heavy for at 3 bit crumb : \  anyone have any better ideas/expierence? 

It shouldnt be that hard Im guessing your trying to login so in my example ill do a post request then get the cookie header


this example should log you onto this forum and show all Trashing and testing threads (which you need to be logged in to see)

code:

---

var xmlhttp = new ActiveXObject("Microsoft.XMLHTTP");
var cookies;

login("username", "password", getTandT);


function getTandT(){
    xmlhttp.open("GET", "http://shoutbox.menthix.net/forumdisplay.php?fid=18", true);
    xmlhttp.setRequestHeader("Cookie" , cookies.join(";") )
    xmlhttp.onreadystatechange = function() {
        if (xmlhttp.readyState==4) {
            if(xmlhttp.status == 200){
                //look its T and T's source
                //Debug.Trace(xmlhttp.responseText);
               
                //show all threads
                var threads = xmlhttp.responseText.match(/<a href="showthread.php\?tid=([0-9]+)">([^<]*)<\/a>/g);
                for(var i=0;i<threads.length;i++){
                    Debug.Trace(threads[i].match(/<a href="showthread.php\?tid=([0-9]+)">([^<]+)<\/a>/)[2])
                }
            }
        }
    }

    xmlhttp.send();

}


function login(username , password , callback){
    xmlhttp.open("POST", "http://shoutbox.menthix.net/member.php", true);
    //set the request header for post
    xmlhttp.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
   
   
    xmlhttp.onreadystatechange = function() {
        if (xmlhttp.readyState==4) {
            if(xmlhttp.status == 200){
                cookies = get_cookies(xmlhttp.GetAllResponseHeaders());
                callback();
            }
        }
    }
   
    xmlhttp.send("username=" + escape(username) + "&password=" + escape(password) + "&action=do_login");
}



//sometimes the Set-Cookie header is sent twice and getResponseHeader only reads the first... so we have to join it ourselfs
function get_cookies(headers){
    var header = headers.split("\r\n");
    var cookies = [];
    for(var i=0;i<header.length;i++){
        var type = header[i].match(/^Set-Cookie: (.*)$/);
        if(type){
            cookies.push(type[1]);
        }
    }
    return cookies;
}

---

thanks for your help, worked like a charm!  unfortunately, after all that, it turns out that i was smoking crack and didn't need it anyway >.<