Code Comments
Programming Forum and web based access to our favorite programming groups.I am trying to make a post request to yahoo to log me in automaticaly
so that I can scrape the pages I want automaticaly.
I have the following code. Which appears to be loging in properly but
not handling the cookies and Im not sure where it is missing ?
When i log on manually there are about 6 cookies *once the page loads*
after login. But with this I only ever find one cookie.
-------------Code----------------
public void run(Notifier notifier) throws IOException {
//Dynamically register the JSSE provider.
java.security.Security.addProvider(new
com.sun.net.ssl.internal.ssl.Provider());
//Set this property to use Sun's reference implementation of the
HTTPS protocol.
System.setProperty("java.protocol.handler.pkgs",
"com.sun.net.ssl.internal.www.protocol");
URL url = new URL("https://login.yahoo.com/");
HttpURLConnection conn = (HttpURLConnection)url.openConnection();
conn.setDoInput(true);
conn.setDoOutput(true);
conn.setRequestMethod("GET");
conn.setAllowUserInteraction(false);
conn.setUseCaches(false);
conn.setRequestProperty( "User-Agent", "Mozilla/4.05 [en] (X11; I;
SunOS 5.5 sun4m)");
conn.setRequestProperty( "Accept", "image/gif, image/x-xbitmap,
image/jpeg, image/pjpeg, image/png, */*");
conn.setRequestProperty( "Accept-Language", "en");
String cookie = conn.getHeaderField("Set-Cookie");
int index = cookie.indexOf(";");
if(index >= 0) cookie = cookie.substring(0, index);
InputStream rawInStream1 = conn.getInputStream();
// get response
BufferedReader rdr = new BufferedReader(new
InputStreamReader(rawInStream1));
StringBuffer postData = new StringBuffer();
String line;
StringBuffer page = new StringBuffer();
Vector formFields = new Vector();
postData.append("login="+URLEncoder.encode(username));
postData.append("&passwd="+URLEncoder.encode(pass));
String[] splitLine;
while ((line = rdr.readLine()) != null) {
Properties props = new Properties();
if(line.startsWith("<input type=hidden")){
splitLine = line.split("(=\")|=|\"");
//System.out.println(line);
postData.append("&"+splitLine[2]+"="+URLEncoder.encode(splitLine[4]));
}
}
url = new URL("https://login.yahoo.com/");
//System.out.println(postData.toString());
conn = (HttpURLConnection)url.openConnection();
conn.setRequestMethod("POST");
conn.setAllowUserInteraction(false); // you may not ask the user
conn.setDoOutput(true); // we want to send things
conn.setRequestProperty( "User-Agent", "Mozilla/4.05 [en] (X11; I;
SunOS 5.5 sun4m)");
conn.setRequestProperty( "Accept", "image/gif, image/x-xbitmap,
image/jpeg, image/pjpeg, image/png, */*");
conn.setRequestProperty( "Accept-Language", "en");
conn.setRequestProperty( "Content-type",
"application/x-www-form-urlencoded" );
conn.setRequestProperty( "Content-length",
Integer.toString(postData.length()));
conn.setRequestProperty("Cookie",cookie);
//get the output stream to POST our form data
conn.connect();
OutputStream rawOutStream = conn.getOutputStream();
PrintWriter pw = new PrintWriter(rawOutStream);
pw.print(postData.toString());
pw.flush ();
//pw.close ();
// for(int i=0;;i++)
// {
// String header=conn.getHeaderField(i);
// if(header==null)
// break;
// if("set-cookie".equalsIgnoreCase(conn.getHeaderFieldKey(i)))
// System.out.println(header);
// }
cookie = conn.getHeaderField("Set-Cookie");
index = cookie.indexOf(";");
if(index >= 0) cookie = cookie.substring(0, index);
InputStream rawInStream = conn.getInputStream();
// get response
BufferedReader readr = new BufferedReader(new
InputStreamReader(rawInStream));
line = readr.readLine();
while(line != null){
System.out.println(line);
line = readr.readLine();
}
}
Post Follow-up to this message
Show a Printable Version
Email This Page to Someone!
Receive updates to this thread
Powered by vBulletin
Copyright 2000-2006 Jelsoft Enterprises Limited.