Java 以编程方式登录网站将返回登录的HTML
我有下面的Java程序，它应该以编程方式登录到我的学生网站，并返回成绩册的HTML。但是，当我运行它时，会返回实际登录站点的HTML： 我最初尝试将初始POST请求发送到inspect element（）的network选项卡中显示的请求URL，但是，这将返回一个错误，并且没有HTML数据。感谢您的帮助，因为这是我第一次尝试这种性质的东西 更新：在尝试登录过程后，我注意到我获得的cookie只包含“lastvisit=95FD925038EF488AA22719B64FB5C4A3”的内容，但缺少“JSESSION_ID”。我不确定这是否是导致问题的原因 更新#2我编辑了我的代码以符合提供的建议，但是，我仍然得到登录页面的HTML而不是成绩数据页面。另外，我添加了print语句来显示我获得的cookie，我注意到缺少“JsessionID”cookie，而我只获得了“lastvisited”cookie 我的cookie print语句的输出如下（出于实验目的）Java 以编程方式登录网站将返回登录的HTML,java,html,Java,Html,我有下面的Java程序，它应该以编程方式登录到我的学生网站，并返回成绩册的HTML。但是，当我运行它时，会返回实际登录站点的HTML： 我最初尝试将初始POST请求发送到inspect element（）的network选项卡中显示的请求URL，但是，这将返回一个错误，并且没有HTML数据。感谢您的帮助，因为这是我第一次尝试这种性质的东西 更新：在尝试登录过程后，我注意到我获得的cookie只包含“lastvisit=95FD925038EF488AA22719B64FB5C4A3”的内容，但缺
GET URL实际响应返回的html。您需要特定的html，并且可以使用不同的端点实际返回所需的html。以下是一些建议：
此站点正在使用JavaServlet身份验证机制。当您尝试获取任何受密码保护的页面时，服务器会在内部发出一个转发，返回登录页面内容。发布有效用户名/密码后，它会将您转发到请求的URL 请注意，这不是重定向，它由应用程序服务器内部管理，客户端看不到服务器端发生了什么 要使其正常工作，您需要发出一个get命令，以访问您试图检索的页面。服务器以登录页面内容和会话id作为cookie进行响应。您必须保存会话id，并在发布到表单操作时将其发送 由于我没有有效的证书进行测试，我不能保证这会起作用。如果凭据无效，服务器将发出大量重定向，这些重定向最终将更改会话，可能是因为会话已无效并已重新创建 请使用有效凭据尝试此操作，并让我知道它是否有效。如果成功登录时有重定向，则可能需要禁用自动重定向并以编程方式执行，检查会话id的更改。根据应用服务器的不同，它可能会出于安全目的更改会话id 也就是说，当您发布有效凭据时，这可能会起作用
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
public class Scraper {
private static final String USERNAME = "user@test.com";
private static final String PASSWORD = "secret";
private static final String GET_URL = "https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form";
private static final String POST_URL = "https://parents.mtsd.k12.nj.us/genesis/j_security_check";
public static void main(String[] args) {
String cookies = doGet(GET_URL);
doPost(POST_URL, cookies);
};
/**
* Send the initial GET request which will forward to the login page
* and retrieve cookies sent by the server.
* Cookies are formatted according to HTTP specification so they can be
* passed to the next request Cookie header.
* @param getURL URL to get
*/
public static String doGet(String getURL) {
StringBuilder formattedCookies = new StringBuilder();
try {
URL url = new URL(getURL);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
InputStream is = conn.getInputStream();
Map<String, List<String>> headers = conn.getHeaderFields();
List<String> cookies = headers.get("Set-Cookie");
ListIterator<String> it = cookies.listIterator();
while (it.hasNext()) {
String[] parts = it.next().split("; ");
formattedCookies.append(parts[0]);
if (it.hasNext()) {
formattedCookies.append("; ");
}
}
System.out.println("\n\nGET OUTPUT");
printContent(is);
} catch (Exception e) {
System.out.println(e.getLocalizedMessage());
}
return formattedCookies.toString();
}
/**
* Post the form parameters and get page content.
* @param postURL URL to post to
* @param cookies The cookies to send
*/
public static void doPost(String postURL, String cookies) {
try {
String postData = String.format("j_username=%s&j_password=%s",
URLEncoder.encode(USERNAME, "UTF-8"), PASSWORD);
URL url = new URL(postURL);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setInstanceFollowRedirects(true);
conn.setRequestProperty("Cookie", cookies);
OutputStreamWriter out = new OutputStreamWriter(conn.getOutputStream());
out.write(postData);
out.close();
InputStream is = conn.getInputStream();
System.out.println("\n\nPOST OUTPUT");
printContent(is);
} catch (Exception e) {
System.out.println(e.getLocalizedMessage());
}
}
public static void printContent(InputStream is) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
String line;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
}
}
导入java.io.BufferedReader；
导入java.io.DataOutputStream；
导入java.io.IOException；
导入java.io.InputStream；
导入java.io.InputStreamReader；
导入java.io.OutputStreamWriter；
导入java.net.HttpURLConnection；
导入java.net.URL；
导入java.net.urlcoder；
导入java.util.List；
导入java.util.ListIterator；
导入java.util.Map；
公共类刮刀{
私有静态最终字符串用户名=”user@test.com";
私有静态最终字符串PASSWORD=“secret”；
私有静态最终字符串GET_URL=”https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form";
私有静态最终字符串POST_URL=”https://parents.mtsd.k12.nj.us/genesis/j_security_check";
公共静态void main（字符串[]args）{
字符串cookies=doGet（获取URL）；
doPost（发布URL、cookies）；
};
/**
*发送初始GET请求，该请求将转发到登录页面
*并检索服务器发送的cookie。
*Cookie是根据HTTP规范格式化的，因此可以
*传递到下一个请求Cookie头。
*@param getURL要获取的URL
*/
公共静态字符串doGet（字符串getURL）{
StringBuilder formattedCookies=新StringBuilder（）；
试一试{
URL=新URL（getURL）；
HttpURLConnection conn=（HttpURLConnection）url.openConnection（）；
InputStream is=conn.getInputStream（）；
Map headers=conn.getHeaderFields（）；
List cookies=headers.get（“Set Cookie”）；
ListIterator it=cookies.ListIterator（）；
while（it.hasNext（））{
String[]parts=it.next（）.split（；）；
格式化cookies.append（部分[0]）；
if（it.hasNext（））{
格式化cookies.append（“；”）；
}
}
System.out.println（“\n\n设置输出”）；
印刷内容（is）；
}捕获（例外e）{
System.out.println（e.getLocalizedMessage（））；
}
返回格式化的cookies.toString（）；
}
/**
*发布表单参数并获取页面内容。
*@param postURL要发布到的URL
*@param cookies要发送的cookies
*/
公共静态void doPost（字符串姿势、字符串cookie）{
试一试{
String postData=String.format（“j_用户名=%s&j_密码=%s”，
URLEncoder.encode（用户名，“UTF-8”），密码）；
URL=新URL（姿势）；
HttpURLConnection conn=（HttpURLConnection）url.openConnection（）；
conn.setRequestMethod（“POST”）；
连接设置输出（真）；
conn.setInstanceFollowRedirects（真）；
conn.setRequestProperty（“Cookie”，cookies）；
OutputStreamWriter out=新的OutputStreamWriter（conn.getOutputStream（））；
写出（postDa
public class Scraper {
static String formData = "user&pass";
static String link = "https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form";
public static void main (String[] args){
String display = postData(link,formData);
System.out.print(display);
}
public static String postData (String url, String data){
URL link = null;
HttpURLConnection connection = null;
StringBuffer stringBuffer = new StringBuffer();
DataOutputStream dataOutputStream = null;
String document = null;
try {
link = new URL(url);
}catch (Exception e){System.out.print(e);}
try {
connection = (HttpURLConnection) link.openConnection();
connection.setRequestMethod("GET");
}catch (Exception e){System.out.print(e);}
connection.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
connection.setRequestProperty("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
connection.setRequestProperty("Accept-Encoding","gzip, deflate, br");
connection.setRequestProperty("Accept-Language","en-US,en;q=0.8");
connection.setRequestProperty("Cookie","");
connection.setDoInput(true);
connection.setDoOutput(true);
//connection.setInstanceFollowRedirects(true);
//setCookie(connection);
String cookie = connection.getHeaderField("Set-Cookie"); //get cookies for session
//try {
// connection.connect();
//}catch (Exception e){System.out.print(e);}
//try {
// dataOutputStream = new DataOutputStream(connection.getOutputStream());
//}catch (Exception e){System.out.print(e);}
//try {
// IOUtils.write(stringBuffer.toString(),dataOutputStream,"UTF-8");
//}catch (Exception e){System.out.print(e);}
//handle redirects
try {
//post data
String postString = data;
stringBuffer.append(postString);
URL redURL = null;
try {
redURL = new URL("https://parents.mtsd.k12.nj.us/genesis/j_security_check");
}catch (Exception e){System.out.print(e);}
connection = (HttpURLConnection)redURL.openConnection();
connection.setRequestMethod("POST");
connection.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
connection.setRequestProperty("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
connection.setRequestProperty("Accept-Encoding","gzip, deflate, br");
connection.setRequestProperty("Accept-Language","en-US,en;q=0.8");
connection.setRequestProperty("Cookie",cookie);
connection.setDoInput(true);
connection.setDoOutput(true);
connection.setInstanceFollowRedirects(true);
//connection.connect();
}catch (Exception e){System.out.print(e);}
InputStream inputStream = null;
String cookie2 = null;
try {
dataOutputStream = new DataOutputStream(connection.getOutputStream());
}catch (Exception e){System.out.print(e);}
try {
IOUtils.write(stringBuffer.toString(),dataOutputStream,"UTF-8");
System.out.println(stringBuffer.toString());
cookie2 = connection.getHeaderField("Set-Cookie"); //get cookies for session
}catch (Exception e){System.out.print(e);}
URL fLink = null;
try {
fLink = new URL("https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form" + stringBuffer.toString());
}catch (Exception e){System.out.print(e);}
HttpURLConnection conn = null;
try {
conn = (HttpURLConnection) fLink.openConnection();
conn.setRequestMethod("GET");
}catch (Exception e){System.out.print(e);}
conn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
conn.setRequestProperty("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
conn.setRequestProperty("Accept-Encoding","gzip, deflate, br");
conn.setRequestProperty("Accept-Language","en-US,en;q=0.8");
conn.setRequestProperty("Cookie",cookie);
conn.setDoInput(true);
conn.setDoOutput(true);
try {
conn.connect();
inputStream = conn.getInputStream();
document = IOUtils.toString(inputStream,"UTF-8");
} catch (Exception e){System.out.print(e);}
System.out.println("cookie -- " + cookie);
System.out.println("cookie2 -- " + cookie2);
return document;
}
}
cookie -- lastvisit=A1753DA7F2454A03B58DF8CBD39C22C4; Expires=Tue, 27-Mar-2018 18:27:55 GMT
cookie2 -- null
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
public class Scraper {
private static final String USERNAME = "user@test.com";
private static final String PASSWORD = "secret";
private static final String GET_URL = "https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form";
private static final String POST_URL = "https://parents.mtsd.k12.nj.us/genesis/j_security_check";
public static void main(String[] args) {
String cookies = doGet(GET_URL);
doPost(POST_URL, cookies);
};
/**
* Send the initial GET request which will forward to the login page
* and retrieve cookies sent by the server.
* Cookies are formatted according to HTTP specification so they can be
* passed to the next request Cookie header.
* @param getURL URL to get
*/
public static String doGet(String getURL) {
StringBuilder formattedCookies = new StringBuilder();
try {
URL url = new URL(getURL);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
InputStream is = conn.getInputStream();
Map<String, List<String>> headers = conn.getHeaderFields();
List<String> cookies = headers.get("Set-Cookie");
ListIterator<String> it = cookies.listIterator();
while (it.hasNext()) {
String[] parts = it.next().split("; ");
formattedCookies.append(parts[0]);
if (it.hasNext()) {
formattedCookies.append("; ");
}
}
System.out.println("\n\nGET OUTPUT");
printContent(is);
} catch (Exception e) {
System.out.println(e.getLocalizedMessage());
}
return formattedCookies.toString();
}
/**
* Post the form parameters and get page content.
* @param postURL URL to post to
* @param cookies The cookies to send
*/
public static void doPost(String postURL, String cookies) {
try {
String postData = String.format("j_username=%s&j_password=%s",
URLEncoder.encode(USERNAME, "UTF-8"), PASSWORD);
URL url = new URL(postURL);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setInstanceFollowRedirects(true);
conn.setRequestProperty("Cookie", cookies);
OutputStreamWriter out = new OutputStreamWriter(conn.getOutputStream());
out.write(postData);
out.close();
InputStream is = conn.getInputStream();
System.out.println("\n\nPOST OUTPUT");
printContent(is);
} catch (Exception e) {
System.out.println(e.getLocalizedMessage());
}
}
public static void printContent(InputStream is) throws IOException {
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
String line;
while ((line = reader.readLine()) != null) {
System.out.println(line);
}
}
}
public static String postData(String url, String data){
try {
OkHttpClient client = new OkHttpClient().newBuilder()
.connectTimeout(180, TimeUnit.SECONDS)
.readTimeout(300, TimeUnit.SECONDS)
.writeTimeout(300, TimeUnit.SECONDS).build();
MediaType mediaType = MediaType.parse("application/x-www-form-urlencoded");
RequestBody body = RequestBody.create(mediaType, data);
Request request = new Request.Builder()
.url(url)
.post(body)
.addHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
.addHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36")
.addHeader("Accept-Encoding","gzip, deflate, br")
.addHeader("Accept-Language","en-US,en;q=0.8")
.build();
String body = client.newCall(request).execute().body().string();
return body;
}catch(Exception e){ e.printStackTrace(); return “An error occurred”; }
}
List<String> cookies = connection.getHeaderFields().get("Set-Cookie");
for (String cookie : cookies) {
connection.addRequestProperty("Cookie", cookie.split(";", 2)[0]);
}
import java.io.DataOutputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;
import org.apache.commons.io.IOUtils;
public class Scraper {
static String formData = "j_username=user&j_password=pass";
static String link = "https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form";
public static void main(String[] args) {
String display = postData(link, formData);
System.out.print(display);
}
public static String postData(String url, String data) {
URL link = null;
HttpURLConnection connection = null;
StringBuffer stringBuffer = new StringBuffer();
DataOutputStream dataOutputStream = null;
String document = null;
try {
link = new URL(url);
} catch (Exception e) {
System.out.print(e);
}
try {
connection = (HttpURLConnection) link.openConnection();
connection.setRequestMethod("GET");
} catch (Exception e) {
System.out.print(e);
}
connection.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
connection.setRequestProperty("User-Agent",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
connection.setRequestProperty("Accept-Encoding", "gzip, deflate, br");
connection.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
// connection.setRequestProperty("Cookie","JSESSIONID=82714DD38C05837934F515C5E0BFD1E3;
// lastvisit=3626F6FB689948B09CEF3DDA56116B7C");
connection.setDoInput(true);
connection.setDoOutput(true);
// connection.setInstanceFollowRedirects(true);
// setCookie(connection);
// String cookie = connection.getHeaderField("Set-Cookie"); //get
// cookies for session
List<String> cookies = connection.getHeaderFields().get("Set-Cookie");
// try {
// connection.connect();
// }catch (Exception e){System.out.print(e);}
// try {
// dataOutputStream = new
// DataOutputStream(connection.getOutputStream());
// }catch (Exception e){System.out.print(e);}
// try {
// IOUtils.write(stringBuffer.toString(),dataOutputStream,"UTF-8");
// }catch (Exception e){System.out.print(e);}
// handle redirects
try {
// post data
String postString = data;
stringBuffer.append(postString);
URL redURL = null;
try {
redURL = new URL("https://parents.mtsd.k12.nj.us/genesis/j_security_check");
} catch (Exception e) {
System.out.print(e);
}
connection = (HttpURLConnection) redURL.openConnection();
connection.setRequestMethod("POST");
connection.setRequestProperty("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
connection.setRequestProperty("User-Agent",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
connection.setRequestProperty("Accept-Encoding", "gzip, deflate, br");
connection.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
// connection.setRequestProperty("Cookie",cookie);
for (String cookie : cookies) {
connection.addRequestProperty("Cookie", cookie.split(";", 2)[0]);
}
connection.setDoInput(true);
connection.setDoOutput(true);
connection.setInstanceFollowRedirects(true);
// connection.connect();
} catch (Exception e) {
System.out.print(e);
}
InputStream inputStream = null;
String cookie2 = null;
try {
dataOutputStream = new DataOutputStream(connection.getOutputStream());
} catch (Exception e) {
System.out.print(e);
}
try {
IOUtils.write(stringBuffer.toString(), dataOutputStream, "UTF-8");
System.out.println(stringBuffer.toString());
cookie2 = connection.getHeaderField("Set-Cookie"); // get cookies
// for session
} catch (Exception e) {
System.out.print(e);
}
URL fLink = null;
try {
fLink = new URL(
"https://parents.mtsd.k12.nj.us/genesis/parents?tab1=studentdata&tab2=gradebook&tab3=weeklysummary&studentid=100916&action=form"
+ stringBuffer.toString());
} catch (Exception e) {
System.out.print(e);
}
HttpURLConnection conn = null;
try {
conn = (HttpURLConnection) fLink.openConnection();
conn.setRequestMethod("GET");
} catch (Exception e) {
System.out.print(e);
}
conn.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
conn.setRequestProperty("User-Agent",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
conn.setRequestProperty("Accept-Encoding", "gzip, deflate, br");
conn.setRequestProperty("Accept-Language", "en-US,en;q=0.8");
// conn.setRequestProperty("Cookie",cookie);
for (String cookie : cookies) {
conn.addRequestProperty("Cookie", cookie.split(";", 2)[0]);
}
conn.setDoInput(true);
conn.setDoOutput(true);
try {
conn.connect();
inputStream = conn.getInputStream();
document = IOUtils.toString(inputStream, "UTF-8");
} catch (Exception e) {
System.out.print(e);
}
for (String cookie : cookies) {
System.out.println("cookie -- " + cookie);
}
System.out.println("cookie2 -- " + cookie2);
return document;
}
}