A comprehensive secret detection engine for scanning code repositories, files, and data streams for exposed secrets and credentials using multiple detection methods.
Complete Implementation
1. Core Secret Detection Engine
package com.trufflehog.core;
import java.util.*;
import java.util.regex.*;
import java.util.concurrent.*;
import java.nio.file.*;
import java.io.*;
import java.security.MessageDigest;
/**
* Main secret detection engine
*/
public class SecretDetectionEngine {
private final List<SecretDetector> detectors;
private final EntropyAnalyzer entropyAnalyzer;
private final KeywordScanner keywordScanner;
private final PatternMatcher patternMatcher;
private final FileScanner fileScanner;
private final ReportGenerator reportGenerator;
private final ExecutorService scanExecutor;
private final Set<String> excludedPaths;
private final Set<String> scannedHashes;
public SecretDetectionEngine(EngineConfig config) {
this.detectors = new ArrayList<>();
this.entropyAnalyzer = new EntropyAnalyzer(config);
this.keywordScanner = new KeywordScanner(config);
this.patternMatcher = new PatternMatcher(config);
this.fileScanner = new FileScanner(config);
this.reportGenerator = new ReportGenerator(config);
this.scanExecutor = Executors.newFixedThreadPool(config.getThreadPoolSize());
this.excludedPaths = new HashSet<>(config.getExcludedPaths());
this.scannedHashes = ConcurrentHashMap.newKeySet();
initializeDetectors();
}
/**
* Scan a directory recursively for secrets
*/
public ScanResult scanDirectory(Path directory) {
try {
List<SecretFinding> findings = new CopyOnWriteArrayList<>();
List<Path> filesToScan = fileScanner.collectFiles(directory);
List<Future<List<SecretFinding>>> futures = new ArrayList<>();
for (Path file : filesToScan) {
if (shouldScanFile(file)) {
futures.add(scanExecutor.submit(() -> scanFile(file)));
}
}
// Collect results
for (Future<List<SecretFinding>> future : futures) {
try {
findings.addAll(future.get());
} catch (Exception e) {
System.err.println("Error scanning file: " + e.getMessage());
}
}
return reportGenerator.generateReport(directory, findings);
} catch (Exception e) {
throw new SecretDetectionException("Failed to scan directory: " + directory, e);
}
}
/**
* Scan a single file for secrets
*/
public List<SecretFinding> scanFile(Path file) {
try {
String contentHash = calculateFileHash(file);
if (scannedHashes.contains(contentHash)) {
return Collections.emptyList(); // Skip duplicate content
}
scannedHashes.add(contentHash);
List<SecretFinding> findings = new ArrayList<>();
String content = Files.readString(file, java.nio.charset.StandardCharsets.UTF_8);
// Apply all detection methods
findings.addAll(patternMatcher.scan(content, file));
findings.addAll(keywordScanner.scan(content, file));
findings.addAll(entropyAnalyzer.analyze(content, file));
// Apply custom detectors
for (SecretDetector detector : detectors) {
findings.addAll(detector.scan(content, file));
}
// Filter out duplicates and false positives
return filterFindings(findings);
} catch (Exception e) {
System.err.println("Error scanning file " + file + ": " + e.getMessage());
return Collections.emptyList();
}
}
/**
* Scan a string content for secrets
*/
public List<SecretFinding> scanContent(String content, String source) {
try {
List<SecretFinding> findings = new ArrayList<>();
findings.addAll(patternMatcher.scan(content, Path.of(source)));
findings.addAll(keywordScanner.scan(content, Path.of(source)));
findings.addAll(entropyAnalyzer.analyze(content, Path.of(source)));
for (SecretDetector detector : detectors) {
findings.addAll(detector.scan(content, Path.of(source)));
}
return filterFindings(findings);
} catch (Exception e) {
throw new SecretDetectionException("Failed to scan content", e);
}
}
/**
* Scan a git repository for secrets
*/
public ScanResult scanGitRepository(Path repoPath) {
try {
GitScanner gitScanner = new GitScanner();
List<SecretFinding> findings = gitScanner.scanRepository(repoPath);
return reportGenerator.generateReport(repoPath, findings);
} catch (Exception e) {
throw new SecretDetectionException("Failed to scan git repository: " + repoPath, e);
}
}
/**
* Add custom detector
*/
public void addDetector(SecretDetector detector) {
detectors.add(detector);
}
/**
* Remove detector
*/
public void removeDetector(SecretDetector detector) {
detectors.remove(detector);
}
private void initializeDetectors() {
// Add built-in detectors
detectors.add(new ApiKeyDetector());
detectors.add(new DatabaseDetector());
detectors.add(new CloudDetector());
detectors.add(new CryptographyDetector());
detectors.add(new SocialMediaDetector());
detectors.add(new PaymentDetector());
}
private boolean shouldScanFile(Path file) {
String filePath = file.toString();
// Check excluded paths
for (String excludedPath : excludedPaths) {
if (filePath.contains(excludedPath)) {
return false;
}
}
// Check file size (skip large files)
try {
long size = Files.size(file);
if (size > 10 * 1024 * 1024) { // 10MB limit
return false;
}
} catch (IOException e) {
return false;
}
// Check file extension
return fileScanner.isSupportedFileType(file);
}
private String calculateFileHash(Path file) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] fileBytes = Files.readAllBytes(file);
byte[] hash = digest.digest(fileBytes);
return bytesToHex(hash);
} catch (Exception e) {
return file.toString(); // Fallback to file path
}
}
private String bytesToHex(byte[] bytes) {
StringBuilder result = new StringBuilder();
for (byte b : bytes) {
result.append(String.format("%02x", b));
}
return result.toString();
}
private List<SecretFinding> filterFindings(List<SecretFinding> findings) {
// Remove duplicates based on content and location
Set<String> seen = new HashSet<>();
List<SecretFinding> filtered = new ArrayList<>();
for (SecretFinding finding : findings) {
String uniqueKey = finding.getSecret() + ":" + finding.getFilePath() + ":" + finding.getLineNumber();
if (!seen.contains(uniqueKey)) {
seen.add(uniqueKey);
filtered.add(finding);
}
}
// Apply confidence filtering
filtered.removeIf(finding -> finding.getConfidence() < 0.7);
return filtered;
}
public void shutdown() {
scanExecutor.shutdown();
try {
if (!scanExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
scanExecutor.shutdownNow();
}
} catch (InterruptedException e) {
scanExecutor.shutdownNow();
Thread.currentThread().interrupt();
}
}
}
2. Domain Models and Configuration
/**
* Secret detection configuration
*/
public class EngineConfig {
private final int threadPoolSize;
private final double entropyThreshold;
private final int minSecretLength;
private final int maxSecretLength;
private final Set<String> excludedPaths;
private final Map<String, Pattern> customPatterns;
private final boolean enableEntropyCheck;
private final boolean enableKeywordScan;
private final boolean enablePatternMatching;
private EngineConfig(Builder builder) {
this.threadPoolSize = builder.threadPoolSize;
this.entropyThreshold = builder.entropyThreshold;
this.minSecretLength = builder.minSecretLength;
this.maxSecretLength = builder.maxSecretLength;
this.excludedPaths = builder.excludedPaths;
this.customPatterns = builder.customPatterns;
this.enableEntropyCheck = builder.enableEntropyCheck;
this.enableKeywordScan = builder.enableKeywordScan;
this.enablePatternMatching = builder.enablePatternMatching;
}
// Getters
public int getThreadPoolSize() { return threadPoolSize; }
public double getEntropyThreshold() { return entropyThreshold; }
public int getMinSecretLength() { return minSecretLength; }
public int getMaxSecretLength() { return maxSecretLength; }
public Set<String> getExcludedPaths() { return excludedPaths; }
public Map<String, Pattern> getCustomPatterns() { return customPatterns; }
public boolean isEnableEntropyCheck() { return enableEntropyCheck; }
public boolean isEnableKeywordScan() { return enableKeywordScan; }
public boolean isEnablePatternMatching() { return enablePatternMatching; }
public static Builder builder() {
return new Builder();
}
public static class Builder {
private int threadPoolSize = Runtime.getRuntime().availableProcessors();
private double entropyThreshold = 3.5;
private int minSecretLength = 10;
private int maxSecretLength = 100;
private Set<String> excludedPaths = new HashSet<>();
private Map<String, Pattern> customPatterns = new HashMap<>();
private boolean enableEntropyCheck = true;
private boolean enableKeywordScan = true;
private boolean enablePatternMatching = true;
public Builder threadPoolSize(int threadPoolSize) {
this.threadPoolSize = threadPoolSize;
return this;
}
public Builder entropyThreshold(double entropyThreshold) {
this.entropyThreshold = entropyThreshold;
return this;
}
public Builder minSecretLength(int minSecretLength) {
this.minSecretLength = minSecretLength;
return this;
}
public Builder maxSecretLength(int maxSecretLength) {
this.maxSecretLength = maxSecretLength;
return this;
}
public Builder excludedPaths(Set<String> excludedPaths) {
this.excludedPaths = excludedPaths;
return this;
}
public Builder excludedPath(String path) {
this.excludedPaths.add(path);
return this;
}
public Builder customPatterns(Map<String, Pattern> customPatterns) {
this.customPatterns = customPatterns;
return this;
}
public Builder customPattern(String name, String regex) {
this.customPatterns.put(name, Pattern.compile(regex));
return this;
}
public Builder enableEntropyCheck(boolean enableEntropyCheck) {
this.enableEntropyCheck = enableEntropyCheck;
return this;
}
public Builder enableKeywordScan(boolean enableKeywordScan) {
this.enableKeywordScan = enableKeywordScan;
return this;
}
public Builder enablePatternMatching(boolean enablePatternMatching) {
this.enablePatternMatching = enablePatternMatching;
return this;
}
public EngineConfig build() {
return new EngineConfig(this);
}
}
}
/**
* Secret finding
*/
public class SecretFinding {
private final String secret;
private final String secretType;
private final String filePath;
private final int lineNumber;
private final int columnNumber;
private final String context;
private final double confidence;
private final String detectorName;
private final Instant timestamp;
private SecretFinding(Builder builder) {
this.secret = builder.secret;
this.secretType = builder.secretType;
this.filePath = builder.filePath;
this.lineNumber = builder.lineNumber;
this.columnNumber = builder.columnNumber;
this.context = builder.context;
this.confidence = builder.confidence;
this.detectorName = builder.detectorName;
this.timestamp = builder.timestamp;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private String secret;
private String secretType;
private String filePath;
private int lineNumber;
private int columnNumber;
private String context;
private double confidence;
private String detectorName;
private Instant timestamp = Instant.now();
public Builder secret(String secret) {
this.secret = secret;
return this;
}
public Builder secretType(String secretType) {
this.secretType = secretType;
return this;
}
public Builder filePath(String filePath) {
this.filePath = filePath;
return this;
}
public Builder lineNumber(int lineNumber) {
this.lineNumber = lineNumber;
return this;
}
public Builder columnNumber(int columnNumber) {
this.columnNumber = columnNumber;
return this;
}
public Builder context(String context) {
this.context = context;
return this;
}
public Builder confidence(double confidence) {
this.confidence = confidence;
return this;
}
public Builder detectorName(String detectorName) {
this.detectorName = detectorName;
return this;
}
public Builder timestamp(Instant timestamp) {
this.timestamp = timestamp;
return this;
}
public SecretFinding build() {
return new SecretFinding(this);
}
}
// Getters
public String getSecret() { return secret; }
public String getSecretType() { return secretType; }
public String getFilePath() { return filePath; }
public int getLineNumber() { return lineNumber; }
public int getColumnNumber() { return columnNumber; }
public String getContext() { return context; }
public double getConfidence() { return confidence; }
public String getDetectorName() { return detectorName; }
public Instant getTimestamp() { return timestamp; }
}
/**
* Scan result
*/
public class ScanResult {
private final Path scanPath;
private final Instant scanTime;
private final long filesScanned;
private final long secretsFound;
private final List<SecretFinding> findings;
private final Map<String, Long> findingsByType;
private final long scanDurationMs;
private ScanResult(Builder builder) {
this.scanPath = builder.scanPath;
this.scanTime = builder.scanTime;
this.filesScanned = builder.filesScanned;
this.secretsFound = builder.secretsFound;
this.findings = builder.findings;
this.findingsByType = builder.findingsByType;
this.scanDurationMs = builder.scanDurationMs;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private Path scanPath;
private Instant scanTime = Instant.now();
private long filesScanned;
private long secretsFound;
private List<SecretFinding> findings = new ArrayList<>();
private Map<String, Long> findingsByType = new HashMap<>();
private long scanDurationMs;
public Builder scanPath(Path scanPath) {
this.scanPath = scanPath;
return this;
}
public Builder scanTime(Instant scanTime) {
this.scanTime = scanTime;
return this;
}
public Builder filesScanned(long filesScanned) {
this.filesScanned = filesScanned;
return this;
}
public Builder secretsFound(long secretsFound) {
this.secretsFound = secretsFound;
return this;
}
public Builder findings(List<SecretFinding> findings) {
this.findings = findings;
return this;
}
public Builder findingsByType(Map<String, Long> findingsByType) {
this.findingsByType = findingsByType;
return this;
}
public Builder scanDurationMs(long scanDurationMs) {
this.scanDurationMs = scanDurationMs;
return this;
}
public ScanResult build() {
return new ScanResult(this);
}
}
// Getters
public Path getScanPath() { return scanPath; }
public Instant getScanTime() { return scanTime; }
public long getFilesScanned() { return filesScanned; }
public long getSecretsFound() { return secretsFound; }
public List<SecretFinding> getFindings() { return findings; }
public Map<String, Long> getFindingsByType() { return findingsByType; }
public long getScanDurationMs() { return scanDurationMs; }
}
/**
* Secret detector interface
*/
public interface SecretDetector {
String getName();
String getDescription();
List<SecretFinding> scan(String content, Path filePath);
}
3. Pattern-Based Detection
/**
* Pattern-based secret detector
*/
public class PatternMatcher {
private final EngineConfig config;
private final Map<String, Pattern> patterns;
public PatternMatcher(EngineConfig config) {
this.config = config;
this.patterns = new HashMap<>();
initializePatterns();
}
/**
* Scan content with regex patterns
*/
public List<SecretFinding> scan(String content, Path filePath) {
if (!config.isEnablePatternMatching()) {
return Collections.emptyList();
}
List<SecretFinding> findings = new ArrayList<>();
String[] lines = content.split("\n");
for (int lineNum = 0; lineNum < lines.length; lineNum++) {
String line = lines[lineNum];
for (Map.Entry<String, Pattern> entry : patterns.entrySet()) {
String patternName = entry.getKey();
Pattern pattern = entry.getValue();
Matcher matcher = pattern.matcher(line);
while (matcher.find()) {
String secret = matcher.group();
if (isValidSecret(secret)) {
SecretFinding finding = SecretFinding.builder()
.secret(maskSecret(secret))
.secretType(patternName)
.filePath(filePath.toString())
.lineNumber(lineNum + 1)
.columnNumber(matcher.start() + 1)
.context(extractContext(line, matcher.start(), matcher.end()))
.confidence(calculateConfidence(patternName, secret))
.detectorName("PatternMatcher")
.build();
findings.add(finding);
}
}
}
}
return findings;
}
/**
* Add custom pattern
*/
public void addPattern(String name, String regex) {
patterns.put(name, Pattern.compile(regex));
}
/**
* Remove pattern
*/
public void removePattern(String name) {
patterns.remove(name);
}
private void initializePatterns() {
// API Keys
addPattern("AWS_ACCESS_KEY", "AKIA[0-9A-Z]{16}");
addPattern("AWS_SECRET_KEY", "[a-zA-Z0-9+/]{40}");
addPattern("GCP_API_KEY", "AIza[0-9A-Za-z\\-_]{35}");
addPattern("GITHUB_TOKEN", "gh[pousr]_[0-9a-zA-Z]{36}");
addPattern("SLACK_TOKEN", "xox[baprs]-[0-9a-zA-Z]{10,48}");
// Database
addPattern("MONGODB_URI", "mongodb(\\+srv)?://[a-zA-Z0-9-_]+:[a-zA-Z0-9-_]+@[a-zA-Z0-9.-]+/[a-zA-Z0-9-_]+");
addPattern("POSTGRES_URL", "postgres(ql)?://[a-zA-Z0-9-_]+:[a-zA-Z0-9-_]+@[a-zA-Z0-9.-]+/[a-zA-Z0-9-_]+");
addPattern("MYSQL_URL", "mysql://[a-zA-Z0-9-_]+:[a-zA-Z0-9-_]+@[a-zA-Z0-9.-]+/[a-zA-Z0-9-_]+");
// Cryptography
addPattern("RSA_PRIVATE_KEY", "-----BEGIN RSA PRIVATE KEY-----");
addPattern("PRIVATE_KEY", "-----BEGIN PRIVATE KEY-----");
addPattern("OPENSSH_PRIVATE_KEY", "-----BEGIN OPENSSH PRIVATE KEY-----");
addPattern("PGP_PRIVATE_KEY", "-----BEGIN PGP PRIVATE KEY BLOCK-----");
// OAuth
addPattern("OAUTH_TOKEN", "ya29\\.[0-9A-Za-z\\-_]+");
// Payment
addPattern("STRIPE_API_KEY", "sk_(live|test)_[0-9a-zA-Z]{24}");
addPattern("STRIPE_PUBLISHABLE_KEY", "pk_(live|test)_[0-9a-zA-Z]{24}");
addPattern("PAYPAL_CLIENT_ID", "A[BCDE][a-zA-Z0-9]{16}");
addPattern("PAYPAL_CLIENT_SECRET", "E[ABCDEF][a-zA-Z0-9]{48}");
// Social Media
addPattern("FACEBOOK_ACCESS_TOKEN", "EAACEdEose0cBA[0-9A-Za-z]+");
addPattern("TWITTER_ACCESS_TOKEN", "[0-9a-zA-Z]{35,44}");
addPattern("LINKEDIN_ACCESS_TOKEN", "[0-9a-zA-Z]{16}");
// Cloud
addPattern("AZURE_STORAGE_KEY", "[a-zA-Z0-9+/]{88}");
addPattern("DIGITALOCEAN_ACCESS_TOKEN", "dop_v1_[a-f0-9]{64}");
// Add custom patterns from config
patterns.putAll(config.getCustomPatterns());
}
private boolean isValidSecret(String secret) {
if (secret.length() < config.getMinSecretLength() ||
secret.length() > config.getMaxSecretLength()) {
return false;
}
// Check for common false positives
if (isFalsePositive(secret)) {
return false;
}
return true;
}
private boolean isFalsePositive(String secret) {
// Common false positives
String[] falsePositives = {
"EXAMPLE", "SAMPLE", "TEST", "DEMO", "DUMMY", "PLACEHOLDER",
"CHANGEME", "TODO", "FIXME", "XXX", "TEMP"
};
String upperSecret = secret.toUpperCase();
for (String fp : falsePositives) {
if (upperSecret.contains(fp)) {
return true;
}
}
return false;
}
private String maskSecret(String secret) {
if (secret.length() <= 8) {
return "***";
}
String firstFour = secret.substring(0, 4);
String lastFour = secret.substring(secret.length() - 4);
return firstFour + "***" + lastFour;
}
private String extractContext(String line, int start, int end) {
int contextStart = Math.max(0, start - 20);
int contextEnd = Math.min(line.length(), end + 20);
return line.substring(contextStart, contextEnd).trim();
}
private double calculateConfidence(String patternName, String secret) {
double baseConfidence = 0.8;
// Adjust confidence based on pattern type
switch (patternName) {
case "AWS_ACCESS_KEY":
case "GITHUB_TOKEN":
case "SLACK_TOKEN":
baseConfidence = 0.95;
break;
case "RSA_PRIVATE_KEY":
case "PRIVATE_KEY":
baseConfidence = 0.99;
break;
case "STRIPE_API_KEY":
baseConfidence = 0.90;
break;
}
// Adjust based on secret characteristics
if (secret.length() >= 32) {
baseConfidence += 0.05;
}
return Math.min(baseConfidence, 1.0);
}
}
4. Entropy-Based Detection
/**
* Entropy-based secret detector for high-randomness strings
*/
public class EntropyAnalyzer {
private final EngineConfig config;
public EntropyAnalyzer(EngineConfig config) {
this.config = config;
}
/**
* Analyze content for high-entropy strings
*/
public List<SecretFinding> analyze(String content, Path filePath) {
if (!config.isEnableEntropyCheck()) {
return Collections.emptyList();
}
List<SecretFinding> findings = new ArrayList<>();
String[] lines = content.split("\n");
for (int lineNum = 0; lineNum < lines.length; lineNum++) {
String line = lines[lineNum];
List<HighEntropyString> highEntropyStrings = findHighEntropyStrings(line);
for (HighEntropyString hes : highEntropyStrings) {
SecretFinding finding = SecretFinding.builder()
.secret(maskSecret(hes.getString()))
.secretType("HIGH_ENTROPY_STRING")
.filePath(filePath.toString())
.lineNumber(lineNum + 1)
.columnNumber(hes.getStartIndex() + 1)
.context(extractContext(line, hes.getStartIndex(), hes.getEndIndex()))
.confidence(hes.getEntropy() / 8.0) // Normalize to 0-1 range
.detectorName("EntropyAnalyzer")
.build();
findings.add(finding);
}
}
return findings;
}
/**
* Find high-entropy strings in a line
*/
private List<HighEntropyString> findHighEntropyStrings(String line) {
List<HighEntropyString> results = new ArrayList<>();
// Tokenize the line and analyze each potential secret candidate
List<String> candidates = extractSecretCandidates(line);
for (String candidate : candidates) {
double entropy = calculateShannonEntropy(candidate);
if (entropy >= config.getEntropyThreshold() &&
candidate.length() >= config.getMinSecretLength() &&
candidate.length() <= config.getMaxSecretLength()) {
int startIndex = line.indexOf(candidate);
if (startIndex != -1) {
results.add(new HighEntropyString(candidate, entropy, startIndex, startIndex + candidate.length()));
}
}
}
return results;
}
/**
* Extract potential secret candidates from a line
*/
private List<String> extractSecretCandidates(String line) {
List<String> candidates = new ArrayList<>();
// Split by common delimiters
String[] tokens = line.split("[\\s=:\",'`\\[\\]{}()<>]+");
for (String token : tokens) {
// Filter tokens that look like potential secrets
if (isPotentialSecret(token)) {
candidates.add(token);
}
}
return candidates;
}
/**
* Check if a token looks like a potential secret
*/
private boolean isPotentialSecret(String token) {
if (token.length() < config.getMinSecretLength()) {
return false;
}
// Skip tokens that are mostly numeric (likely version numbers, etc.)
if (token.matches("\\d+")) {
return false;
}
// Skip common words and identifiers
if (token.matches("[a-zA-Z_][a-zA-Z0-9_]*") &&
!token.matches(".*[0-9].*") &&
token.length() < 20) {
return false;
}
// Skip URLs and email addresses
if (token.matches("^(https?|ftp)://.*") ||
token.matches("^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$")) {
return false;
}
return true;
}
/**
* Calculate Shannon entropy of a string
*/
public double calculateShannonEntropy(String input) {
if (input == null || input.isEmpty()) {
return 0.0;
}
Map<Character, Integer> frequencyMap = new HashMap<>();
// Count character frequencies
for (char c : input.toCharArray()) {
frequencyMap.put(c, frequencyMap.getOrDefault(c, 0) + 1);
}
// Calculate entropy
double entropy = 0.0;
int length = input.length();
for (int count : frequencyMap.values()) {
double probability = (double) count / length;
entropy -= probability * (Math.log(probability) / Math.log(2));
}
return entropy;
}
/**
* Calculate entropy for different character sets
*/
public double calculateCharacterSetEntropy(String input) {
boolean hasLower = input.matches(".*[a-z].*");
boolean hasUpper = input.matches(".*[A-Z].*");
boolean hasDigit = input.matches(".*[0-9].*");
boolean hasSpecial = input.matches(".*[^a-zA-Z0-9].*");
int charSetSize = 0;
if (hasLower) charSetSize += 26;
if (hasUpper) charSetSize += 26;
if (hasDigit) charSetSize += 10;
if (hasSpecial) charSetSize += 32; // Approximate for common special chars
if (charSetSize == 0) return 0.0;
double maxEntropy = Math.log(charSetSize) / Math.log(2);
double actualEntropy = calculateShannonEntropy(input);
return actualEntropy;
}
private String maskSecret(String secret) {
if (secret.length() <= 8) {
return "***";
}
return secret.substring(0, 4) + "***" + secret.substring(secret.length() - 4);
}
private String extractContext(String line, int start, int end) {
int contextStart = Math.max(0, start - 20);
int contextEnd = Math.min(line.length(), end + 20);
return line.substring(contextStart, contextEnd).trim();
}
}
/**
* High entropy string container
*/
class HighEntropyString {
private final String string;
private final double entropy;
private final int startIndex;
private final int endIndex;
public HighEntropyString(String string, double entropy, int startIndex, int endIndex) {
this.string = string;
this.entropy = entropy;
this.startIndex = startIndex;
this.endIndex = endIndex;
}
// Getters
public String getString() { return string; }
public double getEntropy() { return entropy; }
public int getStartIndex() { return startIndex; }
public int getEndIndex() { return endIndex; }
}
5. Keyword-Based Detection
/**
* Keyword-based secret detector
*/
public class KeywordScanner {
private final EngineConfig config;
private final Set<String> secretKeywords;
private final Set<String> falsePositiveKeywords;
public KeywordScanner(EngineConfig config) {
this.config = config;
this.secretKeywords = new HashSet<>();
this.falsePositiveKeywords = new HashSet<>();
initializeKeywords();
}
/**
* Scan content for secret-related keywords
*/
public List<SecretFinding> scan(String content, Path filePath) {
if (!config.isEnableKeywordScan()) {
return Collections.emptyList();
}
List<SecretFinding> findings = new ArrayList<>();
String[] lines = content.split("\n");
for (int lineNum = 0; lineNum < lines.length; lineNum++) {
String line = lines[lineNum];
for (String keyword : secretKeywords) {
if (line.toLowerCase().contains(keyword.toLowerCase())) {
// Found a keyword, now look for potential secrets nearby
List<SecretFinding> keywordFindings = scanAroundKeyword(line, keyword, lineNum, filePath);
findings.addAll(keywordFindings);
}
}
}
return findings;
}
/**
* Scan around a keyword for potential secrets
*/
private List<SecretFinding> scanAroundKeyword(String line, String keyword, int lineNum, Path filePath) {
List<SecretFinding> findings = new ArrayList<>();
int keywordIndex = line.toLowerCase().indexOf(keyword.toLowerCase());
if (keywordIndex == -1) {
return findings;
}
// Look for assignment patterns around the keyword
String[] assignments = extractAssignments(line, keywordIndex);
for (String assignment : assignments) {
if (looksLikeSecret(assignment)) {
SecretFinding finding = SecretFinding.builder()
.secret(maskSecret(assignment))
.secretType("KEYWORD_BASED_" + keyword.toUpperCase())
.filePath(filePath.toString())
.lineNumber(lineNum + 1)
.columnNumber(line.indexOf(assignment) + 1)
.context(line.trim())
.confidence(0.7)
.detectorName("KeywordScanner")
.build();
findings.add(finding);
}
}
return findings;
}
/**
* Extract potential assignments around a keyword
*/
private String[] extractAssignments(String line, int keywordIndex) {
List<String> assignments = new ArrayList<>();
// Look for common assignment patterns
Pattern assignmentPattern = Pattern.compile(
"[=:]+\\s*([\"'`]?[a-zA-Z0-9+/=_-]{10,100}[\"'`]?)"
);
Matcher matcher = assignmentPattern.matcher(line);
while (matcher.find()) {
String value = matcher.group(1).replaceAll("[\"'`]", "");
if (value.length() >= config.getMinSecretLength()) {
assignments.add(value);
}
}
return assignments.toArray(new String[0]);
}
/**
* Check if a string looks like a secret
*/
private boolean looksLikeSecret(String value) {
if (value.length() < config.getMinSecretLength()) {
return false;
}
// Check for high entropy
EntropyAnalyzer entropyAnalyzer = new EntropyAnalyzer(config);
double entropy = entropyAnalyzer.calculateShannonEntropy(value);
if (entropy >= config.getEntropyThreshold() - 0.5) {
return true;
}
// Check for base64 encoding pattern
if (value.matches("^[A-Za-z0-9+/]+={0,2}$") && value.length() >= 20) {
return true;
}
// Check for hex pattern
if (value.matches("^[a-fA-F0-9]+$") && value.length() >= 32) {
return true;
}
return false;
}
private void initializeKeywords() {
// Common secret-related keywords
secretKeywords.add("password");
secretKeywords.add("secret");
secretKeywords.add("key");
secretKeywords.add("token");
secretKeywords.add("api");
secretKeywords.add("auth");
secretKeywords.add("credential");
secretKeywords.add("private");
secretKeywords.add("access");
secretKeywords.add("aws");
secretKeywords.add("gcp");
secretKeywords.add("azure");
secretKeywords.add("database");
secretKeywords.add("mysql");
secretKeywords.add("postgres");
secretKeywords.add("mongodb");
secretKeywords.add("redis");
secretKeywords.add("stripe");
secretKeywords.add("paypal");
secretKeywords.add("github");
secretKeywords.add("gitlab");
secretKeywords.add("bitbucket");
secretKeywords.add("slack");
secretKeywords.add("discord");
secretKeywords.add("twitter");
secretKeywords.add("facebook");
secretKeywords.add("instagram");
secretKeywords.add("linkedin");
// False positive keywords
falsePositiveKeywords.add("example");
falsePositiveKeywords.add("sample");
falsePositiveKeywords.add("test");
falsePositiveKeywords.add("demo");
falsePositiveKeywords.add("dummy");
falsePositiveKeywords.add("placeholder");
falsePositiveKeywords.add("changeme");
falsePositiveKeywords.add("todo");
falsePositiveKeywords.add("fixme");
}
private String maskSecret(String secret) {
if (secret.length() <= 8) {
return "***";
}
return secret.substring(0, 4) + "***" + secret.substring(secret.length() - 4);
}
}
6. Specialized Detectors
```java
/**
- API Key detector
*/
public class ApiKeyDetector implements SecretDetector {
private final Pattern awsKeyPattern = Pattern.compile("AKIA[0-