MACRO NEPAL
Web Scraper with JSoup in Java: Complete Implementation Guide
awiskar acharya
November 9, 2025
2:35 pm

A comprehensive web scraping solution using JSoup library with advanced features like concurrent scraping, data extraction, and export capabilities.

Dependencies

<dependencies>
<!-- JSoup HTML Parser -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.17.1</version>
</dependency>
<!-- HTTP Client -->
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
<version>5.2.1</version>
</dependency>
<!-- JSON Processing -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.15.2</version>
</dependency>
<!-- CSV Writing -->
<dependency>
<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>5.8</version>
</dependency>
<!-- Logging -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.4.11</version>
</dependency>
<!-- Concurrency Utilities -->
<dependency>
<groupId>io.github.resilience4j</groupId>
<artifactId>resilience4j-retry</artifactId>
<version>2.1.0</version>
</dependency>
</dependencies>

Core Scraping Engine

Example 1: Basic JSoup Scraper

package com.webscraper.core;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class BasicScraper {
private static final Logger logger = LoggerFactory.getLogger(BasicScraper.class);
private final ScraperConfig config;
public BasicScraper() {
this.config = new ScraperConfig();
}
public BasicScraper(ScraperConfig config) {
this.config = config;
}
public Document fetchDocument(String url) throws IOException {
logger.info("Fetching document from: {}", url);
return Jsoup.connect(url)
.userAgent(config.getUserAgent())
.timeout(config.getTimeout())
.referrer(config.getReferrer())
.headers(config.getDefaultHeaders())
.get();
}
public List<String> extractLinks(String url, String cssSelector) throws IOException {
Document doc = fetchDocument(url);
Elements linkElements = doc.select(cssSelector);
List<String> links = new ArrayList<>();
for (Element link : linkElements) {
String href = link.attr("abs:href");
if (!href.isEmpty()) {
links.add(href);
}
}
logger.info("Extracted {} links from {}", links.size(), url);
return links;
}
public Map<String, String> extractTextContent(String url, Map<String, String> selectors) 
throws IOException {
Document doc = fetchDocument(url);
Map<String, String> content = new HashMap<>();
for (Map.Entry<String, String> entry : selectors.entrySet()) {
String key = entry.getKey();
String selector = entry.getValue();
Elements elements = doc.select(selector);
if (!elements.isEmpty()) {
content.put(key, elements.text());
} else {
content.put(key, "");
logger.warn("No elements found for selector: {} on {}", selector, url);
}
}
return content;
}
public List<Map<String, String>> extractTableData(String url, String tableSelector) 
throws IOException {
Document doc = fetchDocument(url);
Element table = doc.selectFirst(tableSelector);
if (table == null) {
logger.warn("No table found with selector: {} on {}", tableSelector, url);
return new ArrayList<>();
}
List<Map<String, String>> tableData = new ArrayList<>();
Elements rows = table.select("tr");
// Extract headers if available
Elements headerCells = rows.first().select("th, td");
List<String> headers = new ArrayList<>();
for (Element header : headerCells) {
headers.add(header.text().trim());
}
// If no headers found, generate generic ones
if (headers.isEmpty()) {
int colCount = rows.first().select("td").size();
for (int i = 0; i < colCount; i++) {
headers.add("Column_" + (i + 1));
}
}
// Extract data rows
for (int i = 1; i < rows.size(); i++) {
Element row = rows.get(i);
Elements cells = row.select("td");
if (cells.size() == headers.size()) {
Map<String, String> rowData = new HashMap<>();
for (int j = 0; j < cells.size(); j++) {
rowData.put(headers.get(j), cells.get(j).text().trim());
}
tableData.add(rowData);
}
}
logger.info("Extracted {} rows from table on {}", tableData.size(), url);
return tableData;
}
public void scrapeAndSave(String url, String outputPath) throws IOException {
Document doc = fetchDocument(url);
// Extract various content types
String title = doc.title();
String bodyText = doc.body().text();
Elements images = doc.select("img[src]");
Elements links = doc.select("a[href]");
// Create scraped data object
ScrapedData data = new ScrapedData();
data.setUrl(url);
data.setTitle(title);
data.setContent(bodyText);
data.setImageCount(images.size());
data.setLinkCount(links.size());
data.setTimestamp(System.currentTimeMillis());
// Save to file
DataExporter exporter = new DataExporter();
exporter.exportToJson(data, outputPath);
logger.info("Scraped data saved to: {}", outputPath);
}
}

Example 2: Advanced Concurrent Scraper

package com.webscraper.core;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
public class ConcurrentScraper {
private static final Logger logger = LoggerFactory.getLogger(ConcurrentScraper.class);
private final ScraperConfig config;
private final ExecutorService executor;
private final AtomicInteger successCount;
private final AtomicInteger failureCount;
public ConcurrentScraper(ScraperConfig config) {
this.config = config;
this.executor = Executors.newFixedThreadPool(config.getMaxConcurrentRequests());
this.successCount = new AtomicInteger(0);
this.failureCount = new AtomicInteger(0);
}
public List<ScrapedData> scrapeUrls(List<String> urls) {
List<ScrapedData> results = new ArrayList<>();
List<Future<ScrapedData>> futures = new ArrayList<>();
logger.info("Starting concurrent scrape of {} URLs", urls.size());
// Submit all scraping tasks
for (String url : urls) {
Future<ScrapedData> future = executor.submit(() -> scrapeSingleUrl(url));
futures.add(future);
}
// Collect results
for (Future<ScrapedData> future : futures) {
try {
ScrapedData result = future.get(config.getTimeout() + 5000, TimeUnit.MILLISECONDS);
if (result != null) {
results.add(result);
successCount.incrementAndGet();
}
} catch (Exception e) {
failureCount.incrementAndGet();
logger.warn("Failed to get scraping result: {}", e.getMessage());
}
}
logger.info("Scraping completed: {} successes, {} failures", 
successCount.get(), failureCount.get());
return results;
}
public ScrapedData scrapeSingleUrl(String url) {
ScrapedData data = new ScrapedData();
data.setUrl(url);
data.setTimestamp(System.currentTimeMillis());
try {
Document doc = Jsoup.connect(url)
.userAgent(config.getUserAgent())
.timeout(config.getTimeout())
.headers(config.getDefaultHeaders())
.get();
// Extract basic information
data.setTitle(doc.title());
data.setContent(doc.body().text());
// Extract metadata
data.setMetaDescription(doc.select("meta[name=description]").attr("content"));
data.setMetaKeywords(doc.select("meta[name=keywords]").attr("content"));
// Extract links
data.setLinks(extractLinks(doc));
data.setImages(extractImages(doc));
// Calculate statistics
data.setWordCount(countWords(data.getContent()));
data.setLinkCount(data.getLinks().size());
data.setImageCount(data.getImages().size());
data.setSuccess(true);
logger.debug("Successfully scraped: {}", url);
} catch (IOException e) {
data.setSuccess(false);
data.setError(e.getMessage());
logger.warn("Failed to scrape {}: {}", url, e.getMessage());
}
return data;
}
private List<String> extractLinks(Document doc) {
List<String> links = new ArrayList<>();
doc.select("a[href]").forEach(link -> {
String href = link.attr("abs:href");
if (!href.isEmpty()) {
links.add(href);
}
});
return links;
}
private List<String> extractImages(Document doc) {
List<String> images = new ArrayList<>();
doc.select("img[src]").forEach(img -> {
String src = img.attr("abs:src");
if (!src.isEmpty()) {
images.add(src);
}
});
return images;
}
private int countWords(String text) {
if (text == null || text.trim().isEmpty()) {
return 0;
}
return text.trim().split("\\s+").length;
}
public void shutdown() {
executor.shutdown();
try {
if (!executor.awaitTermination(60, TimeUnit.SECONDS)) {
executor.shutdownNow();
}
} catch (InterruptedException e) {
executor.shutdownNow();
Thread.currentThread().interrupt();
}
}
public int getSuccessCount() {
return successCount.get();
}
public int getFailureCount() {
return failureCount.get();
}
}

Data Models

Example 3: Scraped Data Model

package com.webscraper.model;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.ArrayList;
import java.util.List;
@JsonInclude(JsonInclude.Include.NON_NULL)
public class ScrapedData {
@JsonProperty("url")
private String url;
@JsonProperty("title")
private String title;
@JsonProperty("content")
private String content;
@JsonProperty("meta_description")
private String metaDescription;
@JsonProperty("meta_keywords")
private String metaKeywords;
@JsonProperty("links")
private List<String> links = new ArrayList<>();
@JsonProperty("images")
private List<String> images = new ArrayList<>();
@JsonProperty("word_count")
private int wordCount;
@JsonProperty("link_count")
private int linkCount;
@JsonProperty("image_count")
private int imageCount;
@JsonProperty("timestamp")
private long timestamp;
@JsonProperty("success")
private boolean success;
@JsonProperty("error")
private String error;
// Constructors
public ScrapedData() {}
public ScrapedData(String url) {
this.url = url;
this.timestamp = System.currentTimeMillis();
}
// Getters and Setters
public String getUrl() { return url; }
public void setUrl(String url) { this.url = url; }
public String getTitle() { return title; }
public void setTitle(String title) { this.title = title; }
public String getContent() { return content; }
public void setContent(String content) { this.content = content; }
public String getMetaDescription() { return metaDescription; }
public void setMetaDescription(String metaDescription) { this.metaDescription = metaDescription; }
public String getMetaKeywords() { return metaKeywords; }
public void setMetaKeywords(String metaKeywords) { this.metaKeywords = metaKeywords; }
public List<String> getLinks() { return links; }
public void setLinks(List<String> links) { this.links = links; }
public List<String> getImages() { return images; }
public void setImages(List<String> images) { this.images = images; }
public int getWordCount() { return wordCount; }
public void setWordCount(int wordCount) { this.wordCount = wordCount; }
public int getLinkCount() { return linkCount; }
public void setLinkCount(int linkCount) { this.linkCount = linkCount; }
public int getImageCount() { return imageCount; }
public void setImageCount(int imageCount) { this.imageCount = imageCount; }
public long getTimestamp() { return timestamp; }
public void setTimestamp(long timestamp) { this.timestamp = timestamp; }
public boolean isSuccess() { return success; }
public void setSuccess(boolean success) { this.success = success; }
public String getError() { return error; }
public void setError(String error) { this.error = error; }
@Override
public String toString() {
return String.format(
"ScrapedData{url='%s', title='%s', wordCount=%d, links=%d, images=%d, success=%s}",
url, title, wordCount, linkCount, imageCount, success
);
}
}

Example 4: Scraper Configuration

package com.webscraper.config;
import java.util.HashMap;
import java.util.Map;
public class ScraperConfig {
private String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
private int timeout = 30000; // 30 seconds
private int maxConcurrentRequests = 5;
private String referrer = "https://www.google.com";
private boolean followRedirects = true;
private boolean ignoreHttpErrors = true;
private int maxRetries = 3;
private long delayBetweenRequests = 1000; // 1 second
private Map<String, String> defaultHeaders;
public ScraperConfig() {
this.defaultHeaders = new HashMap<>();
setupDefaultHeaders();
}
private void setupDefaultHeaders() {
defaultHeaders.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
defaultHeaders.put("Accept-Language", "en-US,en;q=0.5");
defaultHeaders.put("Accept-Encoding", "gzip, deflate");
defaultHeaders.put("Connection", "keep-alive");
defaultHeaders.put("Upgrade-Insecure-Requests", "1");
}
// Getters and Setters
public String getUserAgent() { return userAgent; }
public void setUserAgent(String userAgent) { this.userAgent = userAgent; }
public int getTimeout() { return timeout; }
public void setTimeout(int timeout) { this.timeout = timeout; }
public int getMaxConcurrentRequests() { return maxConcurrentRequests; }
public void setMaxConcurrentRequests(int maxConcurrentRequests) { this.maxConcurrentRequests = maxConcurrentRequests; }
public String getReferrer() { return referrer; }
public void setReferrer(String referrer) { this.referrer = referrer; }
public boolean isFollowRedirects() { return followRedirects; }
public void setFollowRedirects(boolean followRedirects) { this.followRedirects = followRedirects; }
public boolean isIgnoreHttpErrors() { return ignoreHttpErrors; }
public void setIgnoreHttpErrors(boolean ignoreHttpErrors) { this.ignoreHttpErrors = ignoreHttpErrors; }
public int getMaxRetries() { return maxRetries; }
public void setMaxRetries(int maxRetries) { this.maxRetries = maxRetries; }
public long getDelayBetweenRequests() { return delayBetweenRequests; }
public void setDelayBetweenRequests(long delayBetweenRequests) { this.delayBetweenRequests = delayBetweenRequests; }
public Map<String, String> getDefaultHeaders() { return defaultHeaders; }
public void setDefaultHeaders(Map<String, String> defaultHeaders) { this.defaultHeaders = defaultHeaders; }
public void addHeader(String name, String value) {
this.defaultHeaders.put(name, value);
}
}

Specialized Scrapers

Example 5: E-commerce Product Scraper

package com.webscraper.specialized;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class EcommerceScraper {
private static final Logger logger = LoggerFactory.getLogger(EcommerceScraper.class);
private final ScraperConfig config;
public EcommerceScraper(ScraperConfig config) {
this.config = config;
}
public Product extractProduct(String url) throws IOException {
Document doc = Jsoup.connect(url)
.userAgent(config.getUserAgent())
.timeout(config.getTimeout())
.get();
Product product = new Product();
product.setUrl(url);
// Extract product name
product.setName(extractProductName(doc));
// Extract price
product.setPrice(extractPrice(doc));
product.setCurrency(extractCurrency(doc));
// Extract description
product.setDescription(extractDescription(doc));
// Extract images
product.setImages(extractProductImages(doc));
// Extract specifications
product.setSpecifications(extractSpecifications(doc));
// Extract availability
product.setAvailable(extractAvailability(doc));
// Extract rating and reviews
product.setRating(extractRating(doc));
product.setReviewCount(extractReviewCount(doc));
product.setTimestamp(System.currentTimeMillis());
logger.info("Extracted product: {}", product.getName());
return product;
}
public List<Product> extractProductsFromListing(String listingUrl, String productSelector) 
throws IOException {
Document doc = Jsoup.connect(listingUrl)
.userAgent(config.getUserAgent())
.timeout(config.getTimeout())
.get();
List<Product> products = new ArrayList<>();
Elements productElements = doc.select(productSelector);
for (Element productElement : productElements) {
Product product = new Product();
// Extract product link
Element linkElement = productElement.selectFirst("a[href]");
if (linkElement != null) {
String productUrl = linkElement.attr("abs:href");
product.setUrl(productUrl);
}
// Extract product name
Element nameElement = productElement.selectFirst(".product-name, .title, h3, h4");
if (nameElement != null) {
product.setName(nameElement.text());
}
// Extract price
String priceText = productElement.selectFirst(".price, .cost, .amount").text();
product.setPrice(parsePrice(priceText));
// Extract image
Element imageElement = productElement.selectFirst("img[src]");
if (imageElement != null) {
product.getImages().add(imageElement.attr("abs:src"));
}
products.add(product);
}
logger.info("Extracted {} products from listing", products.size());
return products;
}
private String extractProductName(Document doc) {
// Try multiple selectors for product name
String[] selectors = {
"h1.product-title", "h1.product-name", "h1.title", 
"meta[property='og:title']", "title"
};
for (String selector : selectors) {
Element element = doc.selectFirst(selector);
if (element != null) {
if (selector.startsWith("meta")) {
return element.attr("content");
}
return element.text();
}
}
return "";
}
private Double extractPrice(Document doc) {
// Try multiple selectors for price
String[] selectors = {
".price", ".product-price", ".cost", ".amount", 
"[itemprop=price]", ".current-price"
};
for (String selector : selectors) {
Element element = doc.selectFirst(selector);
if (element != null) {
return parsePrice(element.text());
}
}
return null;
}
private Double parsePrice(String priceText) {
if (priceText == null || priceText.trim().isEmpty()) {
return null;
}
// Regex to extract numeric price value
Pattern pattern = Pattern.compile("([0-9]+[.,]?[0-9]*)");
Matcher matcher = pattern.matcher(priceText);
if (matcher.find()) {
String priceStr = matcher.group(1).replace(",", ".");
try {
return Double.parseDouble(priceStr);
} catch (NumberFormatException e) {
logger.warn("Failed to parse price: {}", priceText);
}
}
return null;
}
private String extractCurrency(Document doc) {
// Try to extract currency from price or meta tags
String[] priceSelectors = {".price", ".product-price", "[itemprop=price]"};
for (String selector : priceSelectors) {
Element element = doc.selectFirst(selector);
if (element != null) {
String priceText = element.text();
if (priceText.contains("$")) return "USD";
if (priceText.contains("€")) return "EUR";
if (priceText.contains("£")) return "GBP";
}
}
// Check meta tags for currency
Element currencyMeta = doc.selectFirst("meta[property='price:currency']");
if (currencyMeta != null) {
return currencyMeta.attr("content");
}
return "USD"; // Default
}
private String extractDescription(Document doc) {
String[] selectors = {
".product-description", ".description", "[itemprop=description]",
".product-details", "#description"
};
for (String selector : selectors) {
Element element = doc.selectFirst(selector);
if (element != null) {
return element.text();
}
}
return "";
}
private List<String> extractProductImages(Document doc) {
List<String> images = new ArrayList<>();
String[] selectors = {
".product-image img", ".gallery img", "[itemprop=image]",
".main-image img", ".product-photo img"
};
for (String selector : selectors) {
Elements imgElements = doc.select(selector);
for (Element img : imgElements) {
String src = img.attr("abs:src");
if (!src.isEmpty() && !images.contains(src)) {
images.add(src);
}
}
}
return images;
}
private Map<String, String> extractSpecifications(Document doc) {
Map<String, String> specs = new HashMap<>();
// Try table-based specifications
Elements specRows = doc.select("table.specifications tr, table.features tr");
for (Element row : specRows) {
Elements cells = row.select("td, th");
if (cells.size() >= 2) {
String key = cells.get(0).text().replace(":", "").trim();
String value = cells.get(1).text().trim();
specs.put(key, value);
}
}
// Try list-based specifications
Elements specItems = doc.select(".specifications li, .features li");
for (Element item : specItems) {
String text = item.text();
if (text.contains(":")) {
String[] parts = text.split(":", 2);
if (parts.length == 2) {
specs.put(parts[0].trim(), parts[1].trim());
}
}
}
return specs;
}
private boolean extractAvailability(Document doc) {
String[] inStockSelectors = {
".in-stock", ".available", "[itemprop=availability]",
".stock", ".availability"
};
for (String selector : inStockSelectors) {
Element element = doc.selectFirst(selector);
if (element != null) {
String text = element.text().toLowerCase();
return !text.contains("out of stock") && 
!text.contains("unavailable") && 
!text.contains("sold out");
}
}
return true; // Assume available if not specified
}
private Double extractRating(Document doc) {
// Try to extract rating from various structures
String[] ratingSelectors = {
"[itemprop=ratingValue]", ".rating", ".product-rating",
".star-rating", ".review-rating"
};
for (String selector : ratingSelectors) {
Element element = doc.selectFirst(selector);
if (element != null) {
String ratingText = element.text();
try {
return Double.parseDouble(ratingText);
} catch (NumberFormatException e) {
// Try to extract from class names (e.g., star ratings)
String classAttr = element.className();
if (classAttr.matches(".*[0-9]+.*")) {
Pattern pattern = Pattern.compile("([0-9]+)");
Matcher matcher = pattern.matcher(classAttr);
if (matcher.find()) {
return Double.parseDouble(matcher.group(1));
}
}
}
}
}
return null;
}
private Integer extractReviewCount(Document doc) {
String[] reviewSelectors = {
"[itemprop=reviewCount]", ".review-count", ".rating-count"
};
for (String selector : reviewSelectors) {
Element element = doc.selectFirst(selector);
if (element != null) {
String countText = element.text();
try {
return Integer.parseInt(countText.replaceAll("[^0-9]", ""));
} catch (NumberFormatException e) {
logger.warn("Failed to parse review count: {}", countText);
}
}
}
return null;
}
}

Example 6: Product Data Model

package com.webscraper.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class Product {
@JsonProperty("name")
private String name;
@JsonProperty("url")
private String url;
@JsonProperty("price")
private Double price;
@JsonProperty("currency")
private String currency;
@JsonProperty("description")
private String description;
@JsonProperty("images")
private List<String> images = new ArrayList<>();
@JsonProperty("specifications")
private Map<String, String> specifications = new HashMap<>();
@JsonProperty("available")
private boolean available;
@JsonProperty("rating")
private Double rating;
@JsonProperty("review_count")
private Integer reviewCount;
@JsonProperty("timestamp")
private long timestamp;
// Getters and Setters
public String getName() { return name; }
public void setName(String name) { this.name = name; }
public String getUrl() { return url; }
public void setUrl(String url) { this.url = url; }
public Double getPrice() { return price; }
public void setPrice(Double price) { this.price = price; }
public String getCurrency() { return currency; }
public void setCurrency(String currency) { this.currency = currency; }
public String getDescription() { return description; }
public void setDescription(String description) { this.description = description; }
public List<String> getImages() { return images; }
public void setImages(List<String> images) { this.images = images; }
public Map<String, String> getSpecifications() { return specifications; }
public void setSpecifications(Map<String, String> specifications) { this.specifications = specifications; }
public boolean isAvailable() { return available; }
public void setAvailable(boolean available) { this.available = available; }
public Double getRating() { return rating; }
public void setRating(Double rating) { this.rating = rating; }
public Integer getReviewCount() { return reviewCount; }
public void setReviewCount(Integer reviewCount) { this.reviewCount = reviewCount; }
public long getTimestamp() { return timestamp; }
public void setTimestamp(long timestamp) { this.timestamp = timestamp; }
@Override
public String toString() {
return String.format(
"Product{name='%s', price=%s%s, available=%s, rating=%s}",
name, price, currency, available, rating
);
}
}

Data Exporters

Example 7: Multi-Format Data Exporter

package com.webscraper.export;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.opencsv.CSVWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
public class DataExporter {
private static final Logger logger = LoggerFactory.getLogger(DataExporter.class);
private final ObjectMapper objectMapper;
public DataExporter() {
this.objectMapper = new ObjectMapper();
this.objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
}
public void exportToJson(List<?> data, String filePath) throws IOException {
ensureDirectoryExists(filePath);
objectMapper.writeValue(Paths.get(filePath).toFile(), data);
logger.info("Exported {} items to JSON: {}", data.size(), filePath);
}
public void exportToCsv(List<ScrapedData> data, String filePath) throws IOException {
ensureDirectoryExists(filePath);
try (CSVWriter writer = new CSVWriter(new FileWriter(filePath))) {
// Write header
String[] header = {
"URL", "Title", "Word Count", "Link Count", "Image Count", 
"Timestamp", "Success", "Error"
};
writer.writeNext(header);
// Write data
for (ScrapedData item : data) {
String[] row = {
item.getUrl(),
item.getTitle(),
String.valueOf(item.getWordCount()),
String.valueOf(item.getLinkCount()),
String.valueOf(item.getImageCount()),
String.valueOf(item.getTimestamp()),
String.valueOf(item.isSuccess()),
item.getError() != null ? item.getError() : ""
};
writer.writeNext(row);
}
}
logger.info("Exported {} items to CSV: {}", data.size(), filePath);
}
public void exportProductsToCsv(List<Product> products, String filePath) throws IOException {
ensureDirectoryExists(filePath);
try (CSVWriter writer = new CSVWriter(new FileWriter(filePath))) {
// Write header
String[] header = {
"Name", "URL", "Price", "Currency", "Available", 
"Rating", "Review Count", "Image Count"
};
writer.writeNext(header);
// Write data
for (Product product : products) {
String[] row = {
product.getName(),
product.getUrl(),
product.getPrice() != null ? String.valueOf(product.getPrice()) : "",
product.getCurrency(),
String.valueOf(product.isAvailable()),
product.getRating() != null ? String.valueOf(product.getRating()) : "",
product.getReviewCount() != null ? String.valueOf(product.getReviewCount()) : "",
String.valueOf(product.getImages().size())
};
writer.writeNext(row);
}
}
logger.info("Exported {} products to CSV: {}", products.size(), filePath);
}
public void exportToText(List<ScrapedData> data, String filePath) throws IOException {
ensureDirectoryExists(filePath);
StringBuilder content = new StringBuilder();
content.append("Web Scraping Results\n");
content.append("====================\n\n");
for (ScrapedData item : data) {
content.append("URL: ").append(item.getUrl()).append("\n");
content.append("Title: ").append(item.getTitle()).append("\n");
content.append("Word Count: ").append(item.getWordCount()).append("\n");
content.append("Links: ").append(item.getLinkCount()).append("\n");
content.append("Images: ").append(item.getImageCount()).append("\n");
content.append("Success: ").append(item.isSuccess()).append("\n");
if (item.getError() != null) {
content.append("Error: ").append(item.getError()).append("\n");
}
content.append("\n").append("-".repeat(80)).append("\n\n");
}
Files.write(Paths.get(filePath), content.toString().getBytes());
logger.info("Exported {} items to text: {}", data.size(), filePath);
}
private void ensureDirectoryExists(String filePath) throws IOException {
Path path = Paths.get(filePath);
Path parentDir = path.getParent();
if (parentDir != null && !Files.exists(parentDir)) {
Files.createDirectories(parentDir);
}
}
public String generateFilename(String prefix, String extension) {
String timestamp = String.valueOf(System.currentTimeMillis());
return String.format("%s_%s.%s", prefix, timestamp, extension);
}
}

Utility Classes

Example 8: URL Utilities and Validators

package com.webscraper.util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
public class UrlUtils {
private static final Logger logger = LoggerFactory.getLogger(UrlUtils.class);
private static final Pattern URL_PATTERN = Pattern.compile(
"^(https?|ftp)://[\\w\\-]+(\\.[\\w\\-]+)+([\\w\\-.,@?^=%&:/~+#]*[\\w\\-@?^=%&/~+#])?$"
);
public static boolean isValidUrl(String url) {
if (url == null || url.trim().isEmpty()) {
return false;
}
try {
new URL(url);
return URL_PATTERN.matcher(url).matches();
} catch (MalformedURLException e) {
return false;
}
}
public static String normalizeUrl(String url) {
if (url == null) return null;
url = url.trim();
if (!url.startsWith("http://") && !url.startsWith("https://")) {
url = "https://" + url;
}
return url;
}
public static String getDomain(String url) {
try {
URL parsedUrl = new URL(url);
return parsedUrl.getHost();
} catch (MalformedURLException e) {
logger.warn("Failed to parse domain from URL: {}", url);
return null;
}
}
public static List<String> filterUrlsByDomain(List<String> urls, String domain) {
List<String> filtered = new ArrayList<>();
for (String url : urls) {
if (getDomain(url).equals(domain)) {
filtered.add(url);
}
}
return filtered;
}
public static List<String> removeDuplicates(List<String> urls) {
return new ArrayList<>(new LinkedHashSet<>(urls));
}
public static boolean isSameDomain(String url1, String url2) {
String domain1 = getDomain(url1);
String domain2 = getDomain(url2);
return domain1 != null && domain1.equals(domain2);
}
}

Main Application Class

Example 9: Command Line Interface

package com.webscraper;
import com.webscraper.config.ScraperConfig;
import com.webscraper.core.ConcurrentScraper;
import com.webscraper.export.DataExporter;
import com.webscraper.model.ScrapedData;
import com.webscraper.util.UrlUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Scanner;
public class WebScraperCLI {
private static final Logger logger = LoggerFactory.getLogger(WebScraperCLI.class);
private final ScraperConfig config;
private final ConcurrentScraper scraper;
private final DataExporter exporter;
public WebScraperCLI() {
this.config = new ScraperConfig();
this.scraper = new ConcurrentScraper(config);
this.exporter = new DataExporter();
}
public static void main(String[] args) {
WebScraperCLI cli = new WebScraperCLI();
cli.run();
}
public void run() {
Scanner scanner = new Scanner(System.in);
System.out.println("=== Web Scraper with JSoup ===");
System.out.println("1. Scrape single URL");
System.out.println("2. Scrape multiple URLs from file");
System.out.println("3. Scrape with sitemap");
System.out.println("4. Configure settings");
System.out.println("5. Exit");
while (true) {
System.out.print("\nChoose option (1-5): ");
String choice = scanner.nextLine();
switch (choice) {
case "1":
scrapeSingleUrl(scanner);
break;
case "2":
scrapeFromFile(scanner);
break;
case "3":
scrapeWithSitemap(scanner);
break;
case "4":
configureSettings(scanner);
break;
case "5":
System.out.println("Goodbye!");
scraper.shutdown();
return;
default:
System.out.println("Invalid option. Please try again.");
}
}
}
private void scrapeSingleUrl(Scanner scanner) {
System.out.print("Enter URL to scrape: ");
String url = scanner.nextLine();
if (!UrlUtils.isValidUrl(url)) {
System.out.println("Invalid URL format.");
return;
}
try {
List<String> urls = List.of(url);
List<ScrapedData> results = scraper.scrapeUrls(urls);
if (!results.isEmpty()) {
exportResults(results, scanner);
}
} catch (Exception e) {
logger.error("Error during scraping", e);
System.out.println("Error: " + e.getMessage());
}
}
private void scrapeFromFile(Scanner scanner) {
System.out.print("Enter path to URL file: ");
String filePath = scanner.nextLine();
try {
List<String> urls = Files.readAllLines(Paths.get(filePath));
urls = urls.stream()
.filter(UrlUtils::isValidUrl)
.collect(Collectors.toList());
if (urls.isEmpty()) {
System.out.println("No valid URLs found in file.");
return;
}
System.out.printf("Found %d valid URLs. Starting scrape...\n", urls.size());
List<ScrapedData> results = scraper.scrapeUrls(urls);
exportResults(results, scanner);
} catch (IOException e) {
System.out.println("Error reading file: " + e.getMessage());
}
}
private void scrapeWithSitemap(Scanner scanner) {
System.out.print("Enter sitemap URL: ");
String sitemapUrl = scanner.nextLine();
try {
SitemapParser sitemapParser = new SitemapParser(config);
List<String> urls = sitemapParser.parseSitemap(sitemapUrl);
System.out.printf("Found %d URLs in sitemap. Starting scrape...\n", urls.size());
List<ScrapedData> results = scraper.scrapeUrls(urls);
exportResults(results, scanner);
} catch (Exception e) {
System.out.println("Error parsing sitemap: " + e.getMessage());
}
}
private void configureSettings(Scanner scanner) {
System.out.println("\n=== Configuration ===");
System.out.println("1. Set user agent");
System.out.println("2. Set timeout (ms)");
System.out.println("3. Set concurrent requests");
System.out.println("4. Set request delay (ms)");
System.out.println("5. Back to main menu");
System.out.print("Choose option: ");
String choice = scanner.nextLine();
switch (choice) {
case "1":
System.out.print("Enter user agent: ");
config.setUserAgent(scanner.nextLine());
break;
case "2":
System.out.print("Enter timeout (ms): ");
config.setTimeout(Integer.parseInt(scanner.nextLine()));
break;
case "3":
System.out.print("Enter concurrent requests: ");
config.setMaxConcurrentRequests(Integer.parseInt(scanner.nextLine()));
break;
case "4":
System.out.print("Enter request delay (ms): ");
config.setDelayBetweenRequests(Long.parseLong(scanner.nextLine()));
break;
case "5":
return;
default:
System.out.println("Invalid option.");
}
System.out.println("Configuration updated.");
}
private void exportResults(List<ScrapedData> results, Scanner scanner) {
System.out.println("\nScraping completed!");
System.out.printf("Success: %d, Failures: %d\n", 
scraper.getSuccessCount(), scraper.getFailureCount());
System.out.println("\nExport options:");
System.out.println("1. JSON");
System.out.println("2. CSV");
System.out.println("3. Text");
System.out.println("4. Don't export");
System.out.print("Choose format: ");
String format = scanner.nextLine();
if (format.equals("4")) {
return;
}
System.out.print("Enter output file path: ");
String outputPath = scanner.nextLine();
try {
switch (format) {
case "1":
exporter.exportToJson(results, outputPath);
break;
case "2":
exporter.exportToCsv(results, outputPath);
break;
case "3":
exporter.exportToText(results, outputPath);
break;
default:
System.out.println("Invalid format.");
return;
}
System.out.println("Results exported successfully to: " + outputPath);
} catch (IOException e) {
System.out.println("Error exporting results: " + e.getMessage());
}
}
}

Advanced Features

Example 10: Sitemap Parser

package com.webscraper.core;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class SitemapParser {
private static final Logger logger = LoggerFactory.getLogger(SitemapParser.class);
private final ScraperConfig config;
public SitemapParser(ScraperConfig config) {
this.config = config;
}
public List<String> parseSitemap(String sitemapUrl) throws IOException {
List<String> urls = new ArrayList<>();
Document doc = Jsoup.connect(sitemapUrl)
.userAgent(config.getUserAgent())
.timeout(config.getTimeout())
.ignoreContentType(true) // Sitemaps are XML
.get();
// Check if it's a sitemap index
if (doc.select("sitemapindex").size() > 0) {
urls.addAll(parseSitemapIndex(doc));
} else {
urls.addAll(parseUrlset(doc));
}
logger.info("Parsed {} URLs from sitemap: {}", urls.size(), sitemapUrl);
return urls;
}
private List<String> parseSitemapIndex(Document doc) {
List<String> sitemapUrls = new ArrayList<>();
Elements sitemapElements = doc.select("sitemap > loc");
for (Element sitemap : sitemapElements) {
String sitemapUrl = sitemap.text();
try {
sitemapUrls.addAll(parseSitemap(sitemapUrl));
} catch (IOException e) {
logger.warn("Failed to parse nested sitemap: {}", sitemapUrl);
}
}
return sitemapUrls;
}
private List<String> parseUrlset(Document doc) {
List<String> urls = new ArrayList<>();
Elements urlElements = doc.select("url > loc");
for (Element urlElement : urlElements) {
urls.add(urlElement.text());
}
return urls;
}
public List<String> discoverSitemaps(String domainUrl) throws IOException {
List<String> sitemapUrls = new ArrayList<>();
String[] commonSitemapPaths = {
"/sitemap.xml",
"/sitemap_index.xml",
"/sitemap.php",
"/sitemap.txt",
"/robots.txt"
};
for (String path : commonSitemapPaths) {
String sitemapUrl = domainUrl + path;
try {
Jsoup.connect(sitemapUrl)
.userAgent(config.getUserAgent())
.timeout(5000)
.execute();
sitemapUrls.add(sitemapUrl);
logger.info("Found sitemap: {}", sitemapUrl);
} catch (IOException e) {
// Sitemap not found at this path
}
}
// Also check robots.txt for sitemap directives
try {
Document robots = Jsoup.connect(domainUrl + "/robots.txt")
.userAgent(config.getUserAgent())
.timeout(5000)
.get();
String robotsContent = robots.body().text();
String[] lines = robotsContent.split("\n");
for (String line : lines) {
if (line.toLowerCase().startsWith("sitemap:")) {
String sitemapUrl = line.substring(8).trim();
sitemapUrls.add(sitemapUrl);
logger.info("Found sitemap in robots.txt: {}", sitemapUrl);
}
}
} catch (IOException e) {
// robots.txt not found or inaccessible
}
return sitemapUrls;
}
}

Features Summary

This Web Scraper with JSoup provides:

  1. Basic Scraping - Extract text, links, images, and tables
  2. Concurrent Processing - Scrape multiple URLs simultaneously
  3. E-commerce Focus - Specialized product data extraction
  4. Multiple Export Formats - JSON, CSV, and text
  5. Sitemap Support - Automatic discovery and parsing of sitemaps
  6. Configurable Settings - Customizable timeouts, user agents, and headers
  7. Error Handling - Robust error handling and retry mechanisms
  8. Data Validation - URL validation and data cleaning

The scraper is extensible and can be enhanced with features like:

  • JavaScript rendering support
  • CAPTCHA solving
  • Proxy rotation
  • Rate limiting
  • Database storage
  • Web interface
  • API endpoints
  • Scheduled scraping
  • Advanced data analysis
Category : JAVA

Leave a Reply

Your email address will not be published. Required fields are marked *


Macro Nepal Helper