You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
81 lines
2.1 KiB
81 lines
2.1 KiB
package net.jrtechs.www.utils;
|
|
|
|
import java.io.*;
|
|
import java.net.HttpURLConnection;
|
|
import java.net.URL;
|
|
|
|
/**
|
|
* Class which retrieves contents of a website as html
|
|
*
|
|
* @author Jeffery Russell 5-25-18
|
|
*/
|
|
public class WebScraper {
|
|
/**
|
|
* Grabs the contents of a website as a string
|
|
*
|
|
* @param link to open
|
|
* @return source code of website as a single string
|
|
*/
|
|
public static String getWebsite(String link) {
|
|
try {
|
|
URL url = new URL(link);
|
|
|
|
BufferedReader br = new BufferedReader(
|
|
new InputStreamReader(url.openStream())
|
|
);
|
|
return WebScraper.getBufferedReaderData(br);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
return "";
|
|
}
|
|
|
|
|
|
/**
|
|
* Gets contents of a post request sent to a web server
|
|
*
|
|
* @param link to open
|
|
* @param postData to send
|
|
* @return source code of website as a string
|
|
*/
|
|
public static String getPostResponse(String link, String postData) {
|
|
try {
|
|
URL url = new URL(link);
|
|
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
|
conn.setDoOutput(true);
|
|
conn.setRequestMethod("POST");
|
|
|
|
OutputStream os = conn.getOutputStream();
|
|
os.write(postData.getBytes());
|
|
os.flush();
|
|
|
|
BufferedReader br = new BufferedReader(
|
|
new InputStreamReader(conn.getInputStream())
|
|
);
|
|
return WebScraper.getBufferedReaderData(br);
|
|
} catch (Exception ex) {
|
|
ex.printStackTrace();
|
|
}
|
|
|
|
return "";
|
|
}
|
|
|
|
|
|
/**
|
|
* Helper method for getPostResponse() and getWebsite()
|
|
* Slams all contents of a buffered reader into a single string.
|
|
*
|
|
* @param br
|
|
* @return contents of buffered reader
|
|
* @throws IOException -- with br.readLine()
|
|
*/
|
|
private static String getBufferedReaderData(BufferedReader br)
|
|
throws IOException {
|
|
String html = "";
|
|
String line;
|
|
while ((line = br.readLine()) != null) {
|
|
html += line;
|
|
}
|
|
return html;
|
|
}
|
|
}
|