MiningCli.java

/*******************************************************************************
 * Copyright (c) 2025 Carsten Hammer.
 *
 * This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License 2.0
 * which accompanies this distribution, and is available at
 * https://www.eclipse.org/legal/epl-2.0/
 *
 * SPDX-License-Identifier: EPL-2.0
 *
 * Contributors:
 *     Carsten Hammer
 *******************************************************************************/
package org.sandbox.mining.core;

import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;

import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.revwalk.RevCommit;
import org.sandbox.mining.core.candidate.CandidateStatus;
import org.sandbox.mining.core.candidate.CandidateStore;
import org.sandbox.mining.core.candidate.MiningCandidate;
import org.sandbox.mining.core.category.CategoryManager;
import org.sandbox.mining.core.comparison.DeltaReport;
import org.sandbox.mining.core.comparison.ErrorFeedbackCollector;
import org.sandbox.mining.core.comparison.ExternalEvaluationImporter;
import org.sandbox.mining.core.comparison.MiningComparator;
import org.sandbox.mining.core.config.KnownRulesStore;
import org.sandbox.mining.core.config.MiningConfig;
import org.sandbox.mining.core.config.MiningState;
import org.sandbox.mining.core.config.MiningState.DeferredCommit;
import org.sandbox.mining.core.config.MiningState.RepoState;
import org.sandbox.mining.core.config.RepoEntry;
import org.sandbox.mining.core.enrichment.TypeContextEnricher;
import org.sandbox.mining.core.filter.CommitKeywordFilter;
import org.sandbox.jdt.triggerpattern.internal.DslValidator;
import org.sandbox.jdt.triggerpattern.llm.CommitEvaluation;
import org.sandbox.jdt.triggerpattern.llm.DslContextCollector;
import org.sandbox.jdt.triggerpattern.llm.LlmClient;
import org.sandbox.jdt.triggerpattern.llm.LlmClientFactory;
import org.sandbox.jdt.triggerpattern.llm.PromptBuilder;
import org.sandbox.jdt.triggerpattern.llm.PromptBuilder.CommitData;
import org.sandbox.jdt.triggerpattern.git.CommitWalker;
import org.sandbox.jdt.triggerpattern.git.DiffExtractor;
import org.sandbox.jdt.triggerpattern.git.RepoCloner;
import org.sandbox.mining.core.report.GithubPagesGenerator;
import org.sandbox.mining.core.report.JsonReporter;
import org.sandbox.mining.core.report.NetBeansReporter;
import org.sandbox.mining.core.report.ReportAggregator;
import org.sandbox.mining.core.report.StatisticsCollector;

/**
 * CLI entry point for AI-powered commit analysis.
 *
 * <p>Analyzes Eclipse project commits using a configurable LLM provider and
 * generates reports for TriggerPattern DSL mining.</p>
 *
 * <p>Usage: java -jar sandbox-mining-core.jar [options]</p>
 */
public class MiningCli {

/**
 * Output stream for informational messages.
 * Default: {@code System.out}. Set to {@code System.err} when {@code --strict-netbeans} is active,
 * keeping stdout clean for pipe-friendly NetBeans-format output.
 */
private PrintStream miningLog = System.out;
private final Function<String, LlmClient> llmClientFactory;
private int stagedCandidateCount;
private final Map<String, Map<String, CommitEvaluation>> completedEvaluations = new HashMap<>();
private final List<CommitEvaluation> reusedEvaluations = new ArrayList<>();

/**
 * Known rules context for LLM prompts, formatted by {@link KnownRulesStore}.
 * Populated in {@link #run(String[])} and consumed in {@link #processBatch}.
 */
private String knownRulesContext = ""; //$NON-NLS-1$

private static final String OPT_CONFIG = "--config"; //$NON-NLS-1$
private static final String OPT_STATE = "--state"; //$NON-NLS-1$
private static final String OPT_SANDBOX_ROOT = "--sandbox-root"; //$NON-NLS-1$
private static final String OPT_BATCH_SIZE = "--batch-size"; //$NON-NLS-1$
private static final String OPT_OUTPUT = "--output"; //$NON-NLS-1$
private static final String OPT_COMMITS_PER_REQUEST = "--commits-per-request"; //$NON-NLS-1$
private static final String OPT_MAX_FAILURE_DURATION = "--max-failure-duration"; //$NON-NLS-1$
private static final String OPT_LLM_PROVIDER = "--llm-provider"; //$NON-NLS-1$
private static final String OPT_RETRY_DEFERRED = "--retry-deferred"; //$NON-NLS-1$
private static final String OPT_RESET_LEARNED_LIMITS = "--reset-learned-limits"; //$NON-NLS-1$
private static final String OPT_COMMIT_LIST = "--commit-list"; //$NON-NLS-1$
private static final String OPT_MAX_DURATION = "--max-duration"; //$NON-NLS-1$
private static final String OPT_COMPARISON_MODE = "--comparison-mode"; //$NON-NLS-1$
private static final String OPT_COPILOT_RESULTS = "--copilot-results"; //$NON-NLS-1$
private static final String OPT_ENRICH_TYPE_CONTEXT = "--enrich-type-context"; //$NON-NLS-1$
private static final String OPT_KEYWORD_FILTER = "--keyword-filter"; //$NON-NLS-1$
private static final String OPT_OUTPUT_FORMAT = "--output-format"; //$NON-NLS-1$
private static final String OPT_STRICT_NETBEANS = "--strict-netbeans"; //$NON-NLS-1$

private static final int DEFAULT_BATCH_SIZE = 500;
private static final int DEFAULT_COMMITS_PER_REQUEST = 4;
private static final int DEFAULT_MAX_FAILURE_DURATION_SECONDS = 300;

private static final DateTimeFormatter COMMIT_DATE_FORMAT =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneOffset.UTC); //$NON-NLS-1$

/**
 * Default upper bound for useful commits by diff size.
 * <p>Used as a fallback when the configuration does not specify
 * {@code max-diff-lines-per-commit}. The configured value from
 * {@code repos.yml} is preferred.</p>
 */
private static final int MAX_USEFUL_DIFF_LINES = 300;

public MiningCli() {
this(LlmClientFactory::createFromEnvironment);
}

MiningCli(Function<String, LlmClient> llmClientFactory) {
this.llmClientFactory = llmClientFactory;
}

public static void main(String[] args) {
try {
new MiningCli().run(args);
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
e.printStackTrace();
System.exit(1);
}
}

/**
 * Runs the mining process with the given arguments.
 *
 * @param args CLI arguments
 * @throws IOException     if an I/O error occurs
 * @throws GitAPIException if a Git operation fails
 */
public void run(String[] args) throws IOException, GitAPIException {
long startTimeMs = System.currentTimeMillis();
stagedCandidateCount = 0;
completedEvaluations.clear();
reusedEvaluations.clear();
Path configPath = null;
Path statePath = null;
Path sandboxRoot = Path.of("."); //$NON-NLS-1$
int batchSize = DEFAULT_BATCH_SIZE;
int commitsPerRequest = DEFAULT_COMMITS_PER_REQUEST;
int maxFailureDurationSeconds = DEFAULT_MAX_FAILURE_DURATION_SECONDS;
Path outputDir = Path.of("output"); //$NON-NLS-1$
String llmProvider = null;
boolean retryDeferred = false;
boolean resetLearnedLimits = false;
Path commitListPath = null;
int maxDurationMinutes = 0;
boolean comparisonMode = false;
Path copilotResultsPath = null;
boolean enrichTypeContext = false;
Path keywordFilterPath = null;
String outputFormat = "json"; //$NON-NLS-1$
boolean strictNetbeans = false;

for (int i = 0; i < args.length; i++) {
switch (args[i]) {
case OPT_CONFIG:
configPath = Path.of(requireArg(args, ++i, OPT_CONFIG));
break;
case OPT_STATE:
statePath = Path.of(requireArg(args, ++i, OPT_STATE));
break;
case OPT_SANDBOX_ROOT:
sandboxRoot = Path.of(requireArg(args, ++i, OPT_SANDBOX_ROOT));
break;
case OPT_BATCH_SIZE:
batchSize = Integer.parseInt(requireArg(args, ++i, OPT_BATCH_SIZE));
break;
case OPT_COMMITS_PER_REQUEST:
commitsPerRequest = Integer.parseInt(requireArg(args, ++i, OPT_COMMITS_PER_REQUEST));
if (commitsPerRequest < 1) {
throw new IllegalArgumentException(
"--commits-per-request must be >= 1 but was " + commitsPerRequest); //$NON-NLS-1$
}
break;
case OPT_OUTPUT:
outputDir = Path.of(requireArg(args, ++i, OPT_OUTPUT));
break;
case OPT_MAX_FAILURE_DURATION:
maxFailureDurationSeconds = Integer.parseInt(requireArg(args, ++i, OPT_MAX_FAILURE_DURATION));
if (maxFailureDurationSeconds < 10) {
throw new IllegalArgumentException(
"--max-failure-duration must be at least 10 seconds but was " //$NON-NLS-1$
+ maxFailureDurationSeconds);
}
break;
case OPT_LLM_PROVIDER:
llmProvider = requireArg(args, ++i, OPT_LLM_PROVIDER);
break;
case OPT_RETRY_DEFERRED:
retryDeferred = true;
break;
case OPT_RESET_LEARNED_LIMITS:
resetLearnedLimits = true;
break;
case OPT_COMMIT_LIST:
commitListPath = Path.of(requireArg(args, ++i, OPT_COMMIT_LIST));
break;
case OPT_MAX_DURATION:
maxDurationMinutes = Integer.parseInt(requireArg(args, ++i, OPT_MAX_DURATION));
break;
case OPT_COMPARISON_MODE:
comparisonMode = true;
break;
case OPT_COPILOT_RESULTS:
copilotResultsPath = Path.of(requireArg(args, ++i, OPT_COPILOT_RESULTS));
break;
case OPT_ENRICH_TYPE_CONTEXT:
enrichTypeContext = true;
break;
case OPT_KEYWORD_FILTER:
keywordFilterPath = Path.of(requireArg(args, ++i, OPT_KEYWORD_FILTER));
break;
case OPT_OUTPUT_FORMAT:
outputFormat = requireArg(args, ++i, OPT_OUTPUT_FORMAT);
break;
case OPT_STRICT_NETBEANS:
strictNetbeans = true;
break;
default:
System.err.println("Unknown option: " + args[i]); //$NON-NLS-1$
printUsage();
return;
}
}

if (configPath == null) {
configPath = sandboxRoot.resolve(".github/refactoring-mining/repos.yml"); //$NON-NLS-1$
}
if (statePath == null) {
statePath = sandboxRoot.resolve(".github/refactoring-mining/state.json"); //$NON-NLS-1$
}

// Use miningLog for all informational output; when --strict-netbeans is set
// it redirects to stderr so stdout stays clean for pipe-friendly output.
if (strictNetbeans) {
miningLog = System.err;
}

miningLog.println("=== Sandbox Mining Core ==="); //$NON-NLS-1$
miningLog.println("Config: " + configPath); //$NON-NLS-1$
miningLog.println("State:  " + statePath); //$NON-NLS-1$
miningLog.println("Output: " + outputDir); //$NON-NLS-1$

MiningConfig config = MiningConfig.parse(configPath);
MiningState state = MiningState.load(statePath);
MiningState.backup(statePath);
CategoryManager categoryManager = new CategoryManager();
DslContextCollector dslCollector = new DslContextCollector();
String dslContext = dslCollector.collectContext(sandboxRoot);
PromptBuilder promptBuilder = new PromptBuilder();
DslValidator validator = new DslValidator();
StatisticsCollector stats = new StatisticsCollector();
ReportAggregator aggregator = new ReportAggregator();

// Enrich prompt with type context if requested
// The TypeContextEnricher is initialized here but applied per-batch
// in processBatch() where the actual diff content is available.
TypeContextEnricher typeEnricher = enrichTypeContext ? new TypeContextEnricher() : null;

// Collect error feedback from existing evaluations for prompt improvement
JsonReporter feedbackReader = new JsonReporter();
Path existingEvals = outputDir.resolve("evaluations.json"); //$NON-NLS-1$
List<CommitEvaluation> previousEvals = feedbackReader.loadExistingEvaluations(existingEvals);
for (CommitEvaluation evaluation : previousEvals) {
completedEvaluations.computeIfAbsent(evaluation.repoUrl(), key -> new HashMap<>())
.put(evaluation.commitHash(), evaluation);
}
if (!previousEvals.isEmpty()) {
ErrorFeedbackCollector errorCollector = new ErrorFeedbackCollector();
errorCollector.collect(previousEvals);
if (errorCollector.getErrorCount() > 0) {
promptBuilder.setErrorFeedback(errorCollector.formatFeedback());
miningLog.println("Loaded " + errorCollector.getErrorCount() //$NON-NLS-1$
+ " error patterns from previous evaluations for prompt feedback"); //$NON-NLS-1$
}
}

// Load known rules and provide them as LLM context to avoid duplicates
Path knownRulesPath = outputDir.resolve("known-rules.json"); //$NON-NLS-1$
KnownRulesStore knownRules = KnownRulesStore.load(knownRulesPath);
knownRulesContext = knownRules.formatForPrompt();
if (!knownRulesContext.isEmpty()) {
miningLog.println("Loaded " + knownRules.size() + " known rules as LLM context"); //$NON-NLS-1$ //$NON-NLS-2$
}

// Initialize keyword filter if specified
CommitKeywordFilter keywordFilter = null;
if (keywordFilterPath != null) {
keywordFilter = new CommitKeywordFilter(keywordFilterPath);
miningLog.println("Keyword filter loaded: " + keywordFilter.getKeywords().size() + " keywords"); //$NON-NLS-1$ //$NON-NLS-2$
}

// Read commit list if specified — only these commits will be processed
Set<String> commitListHashes = null;
if (commitListPath != null) {
List<String> hashes = readCommitList(commitListPath);
commitListHashes = new HashSet<>(hashes);
miningLog.println("Commit list loaded: " + commitListHashes.size() + " commits to process"); //$NON-NLS-1$ //$NON-NLS-2$
}

Exception miningFailure = null;
Path workDir = Files.createTempDirectory("mining-core-"); //$NON-NLS-1$
try (LlmClient llmClient = llmClientFactory.apply(llmProvider)) {
llmClient.setMaxFailureDuration(Duration.ofSeconds(maxFailureDurationSeconds));
// Reset learned limits if requested or if model changed
for (RepoEntry repo : config.getRepositories()) {
RepoState repoState = state.getRepoState(repo.getUrl());
String currentModel = llmClient.getModel();
if (resetLearnedLimits || (repoState.getLastModelUsed() != null
&& !repoState.getLastModelUsed().equals(currentModel))) {
if (repoState.getLearnedMaxDiffLines() != -1) {
miningLog.println("Resetting learned diff limit for " + repo.getUrl() //$NON-NLS-1$
+ " (was " + repoState.getLearnedMaxDiffLines() + " lines)"); //$NON-NLS-1$ //$NON-NLS-2$
repoState.setLearnedMaxDiffLines(-1);
}
// Reset deferred retry counts on model change
for (DeferredCommit dc : repoState.getDeferredCommits()) {
dc.setRetryCount(0);
}
}
repoState.setLastModelUsed(currentModel);
}
try {
processRepositories(config, state, statePath, workDir, batchSize, commitsPerRequest,
llmClient, promptBuilder, dslContext, categoryManager, validator, stats, aggregator,
startTimeMs, maxDurationMinutes, typeEnricher, keywordFilter, commitListHashes,
outputDir, sandboxRoot);
// Always process deferred commits at end of each run
retryDeferredCommits(state, statePath, workDir, llmClient, promptBuilder,
dslContext, categoryManager, validator, stats, aggregator, config, retryDeferred);
} catch (IOException | GitAPIException e) {
// Write the results that did complete, then propagate the original failure.
miningFailure = e;
} finally {
try {
deleteDirectory(workDir);
} catch (IOException e) {
if (miningFailure == null) {
miningFailure = e;
} else {
miningFailure.addSuppressed(e);
}
}
}
printDeferredReport(state, config);
printRunSummary(stats, state, config, llmClient, startTimeMs);

// Persist run metadata
long durationMs = System.currentTimeMillis() - startTimeMs;
int totalDeferred = 0;
int totalPermanentlySkipped = 0;
for (RepoEntry repo : config.getRepositories()) {
RepoState rs = state.getRepoState(repo.getUrl());
totalDeferred += rs.getDeferredCommits().size();
totalPermanentlySkipped += rs.getPermanentlySkipped().size();
}
String startedAt = Instant.ofEpochMilli(startTimeMs).toString();
String completedAt = Instant.now().toString();
stats.recordRunMetadata(startedAt, completedAt, durationMs / 1000,
llmClient.getClass().getSimpleName(), llmClient.getModel(),
batchSize, commitsPerRequest,
llmClient.getDailyRequestCount(), totalDeferred, totalPermanentlySkipped);
stats.computeTimeWindow(aggregator.getAllEvaluations());
}

// Generate output even when discovery stopped at a failed or incomplete batch.
try {
Files.createDirectories(outputDir);
JsonReporter jsonReporter = new JsonReporter();
jsonReporter.writeEvaluations(aggregator.getAllEvaluations(), outputDir);
jsonReporter.writeStatistics(stats, outputDir);

GithubPagesGenerator pagesGenerator = new GithubPagesGenerator();
pagesGenerator.generate(aggregator.getAllEvaluations(), stats, outputDir);

// Save GREEN+VALID evaluations as staged candidates instead of writing directly
// to productive bundled hint files. Candidates are staged in mining-candidates/
// for review and validation before promotion.
saveRunCandidates(aggregator, validator, sandboxRoot);

// NetBeans format output
if ("netbeans".equals(outputFormat) || "both".equals(outputFormat)) { //$NON-NLS-1$ //$NON-NLS-2$
NetBeansReporter nbReporter = new NetBeansReporter();
nbReporter.write(aggregator.getAllEvaluations(), outputDir);
// With --strict-netbeans, NetBeans format goes to stdout (pipe-friendly),
// everything else goes to stderr via miningLog field
nbReporter.printToStream(aggregator.getAllEvaluations(), System.out);
}

// Comparison mode
if (comparisonMode && copilotResultsPath != null) {
ExternalEvaluationImporter importer = new ExternalEvaluationImporter();
List<CommitEvaluation> copilotResults = importer.importFromJson(copilotResultsPath);
MiningComparator comparator = new MiningComparator();
DeltaReport deltaReport = comparator.compare(aggregator.getAllEvaluations(), copilotResults);
miningLog.println(deltaReport.format());
deltaReport.writeToFiles(outputDir);
miningLog.println("Delta report written to " + outputDir.resolve("delta-report.json")); //$NON-NLS-1$ //$NON-NLS-2$
miningLog.println("Delta report written to " + outputDir.resolve("delta-report.md")); //$NON-NLS-1$ //$NON-NLS-2$
}

// Register GREEN+VALID evaluations in known-rules.json
// Run number is not tracked in MiningState; use globalTotalProcessed as proxy
int newKnownRules = knownRules.registerFromEvaluations(
aggregator.getAllEvaluations(), state.getGlobalTotalProcessed());
if (newKnownRules > 0) {
knownRules.save(knownRulesPath);
miningLog.println("Registered " + newKnownRules + " new known rules (total: " + knownRules.size() + ")"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}

state.save(statePath);
} catch (IOException e) {
if (miningFailure == null) {
throw e;
}
miningFailure.addSuppressed(e);
}

if (miningFailure != null) {
miningLog.println("=== Mining incomplete ==="); //$NON-NLS-1$
if (miningFailure instanceof IOException ioFailure) {
throw ioFailure;
}
throw (GitAPIException) miningFailure;
}

miningLog.println("=== Mining complete ==="); //$NON-NLS-1$
miningLog.println("Processed: " + stats.getTotalProcessed() + " commits"); //$NON-NLS-1$ //$NON-NLS-2$
miningLog.println("Relevant:  " + stats.getRelevant()); //$NON-NLS-1$
miningLog.println("Candidates: " + stagedCandidateCount + " staged in mining-candidates/"); //$NON-NLS-1$ //$NON-NLS-2$
miningLog.println("Output:    " + outputDir.toAbsolutePath()); //$NON-NLS-1$
}

private void processRepositories(MiningConfig config, MiningState state, Path statePath,
Path workDir, int batchSize, int commitsPerRequest,
LlmClient llmClient, PromptBuilder promptBuilder,
String dslContext, CategoryManager categoryManager, DslValidator validator,
StatisticsCollector stats, ReportAggregator aggregator,
long startTimeMs, int maxDurationMinutes,
TypeContextEnricher typeEnricher,
CommitKeywordFilter keywordFilter, Set<String> commitListHashes,
Path outputDir, Path sandboxRoot) throws IOException, GitAPIException {
RepoCloner cloner = new RepoCloner();
// Dynamic batch size tracking (reduced on truncation)
int[] dynamicCPR = { commitsPerRequest };

for (RepoEntry repo : config.getRepositories()) {
if (shouldStop(startTimeMs, maxDurationMinutes)) {
miningLog.println("Max duration reached (" + maxDurationMinutes //$NON-NLS-1$
+ " minutes). Stopping. Will resume from current position on next run."); //$NON-NLS-1$
return;
}
if (!llmClient.hasRemainingQuota()) {
miningLog.println("Daily API quota exhausted (" + llmClient.getDailyRequestCount() //$NON-NLS-1$
+ " requests used). Stopping. Will resume from current position on next run."); //$NON-NLS-1$
return;
}
if (llmClient.isApiUnavailable()) {
logApiUnavailable(llmClient);
throw new IOException("LLM API unavailable; stopping mining"); //$NON-NLS-1$
}
miningLog.println("Processing: " + repo.getUrl()); //$NON-NLS-1$
Path repoDir = workDir.resolve(repoDirectoryName(repo.getUrl()));
cloner.cloneRepo(repo.getUrl(), repo.getBranch(), repoDir);

String lastCommit = state.getLastProcessedCommit(repo.getUrl());
RepoState repoState = state.getRepoState(repo.getUrl());

// Determine epoch-aware date range
String effectiveStartDate = config.getStartDate();
String effectiveEndDate = config.getEndDate();
List<MiningConfig.EpochEntry> epochs = config.getEpochs();
if (!epochs.isEmpty()) {
int epochIdx = repoState.getCurrentEpoch();
if (epochIdx < epochs.size()) {
MiningConfig.EpochEntry epoch = epochs.get(epochIdx);
effectiveStartDate = epoch.getStart();
effectiveEndDate = epoch.getEnd();
miningLog.println("  Epoch " + epochIdx + ": " + epoch); //$NON-NLS-1$ //$NON-NLS-2$
} else {
miningLog.println("  All " + epochs.size() + " epochs completed for " + repo.getUrl()); //$NON-NLS-1$ //$NON-NLS-2$
continue;
}
}

try (CommitWalker walker = new CommitWalker(repoDir);
DiffExtractor diffExtractor = new DiffExtractor(repoDir,
config.getMaxDiffLinesPerCommit(), repo.getPaths(),
config.getMaxFilesPerCommit())) {
List<RevCommit> batch = walker.nextBatch(lastCommit, effectiveStartDate, effectiveEndDate, batchSize);
while (!batch.isEmpty()) {
// Remember the last commit in the unfiltered batch for walker advancement
String lastBatchCommit = batch.get(batch.size() - 1).getName();

// Filter batch by commit-list if specified — commits not in the
// list are silently skipped (state still advances past them)
List<RevCommit> effectiveBatch = batch;
if (commitListHashes != null) {
int beforeSize = batch.size();
effectiveBatch = batch.stream()
.filter(c -> commitListHashes.contains(c.getName()))
.toList();
if (effectiveBatch.size() < beforeSize) {
miningLog.println("  Commit-list filter: " + effectiveBatch.size() //$NON-NLS-1$
+ " of " + beforeSize + " commits matched"); //$NON-NLS-1$
}
if (effectiveBatch.isEmpty()) {
// Advance state past skipped commits and continue to next batch
state.updateLastProcessedCommit(repo.getUrl(), lastBatchCommit);
batch = walker.nextBatch(lastBatchCommit, effectiveStartDate, effectiveEndDate, batchSize);
continue;
}
}
// Group commits into sub-batches for API calls
for (int i = 0; i < effectiveBatch.size();) {
if (shouldStop(startTimeMs, maxDurationMinutes)) {
miningLog.println("Max duration reached. Stopping batch processing."); //$NON-NLS-1$
return;
}
if (!llmClient.hasRemainingQuota()) {
miningLog.println("Daily API quota exhausted (" //$NON-NLS-1$
+ llmClient.getDailyRequestCount()
+ " requests used). Stopping. Will resume from current position on next run."); //$NON-NLS-1$
return;
}
if (llmClient.isApiUnavailable()) {
logApiUnavailable(llmClient);
throw new IOException("LLM API unavailable; stopping mining"); //$NON-NLS-1$
}
int end = Math.min(i + dynamicCPR[0], effectiveBatch.size());
List<RevCommit> subBatch = effectiveBatch.subList(i, end);
try {
processBatch(subBatch, repo, diffExtractor, state, statePath,
llmClient, promptBuilder, dslContext, categoryManager,
validator, stats, aggregator, config.getMinDiffLinesPerCommit(),
config.getMaxDiffLinesPerCommit(), typeEnricher, keywordFilter, outputDir, sandboxRoot);
} catch (IncompleteBatchException e) {
if (dynamicCPR[0] <= 1) {
throw e;
}
// Even a response that ended normally can omit evaluations. Use the same
// bounded smaller-batch retry, stopping if a single-commit batch is incomplete.
dynamicCPR[0] = Math.max(1, dynamicCPR[0] / 2);
// The completed prefix is already checkpointed. Retry from the first gap;
// later completed results are cached and must not be requested again.
i += e.firstMissingIndex;
miningLog.println("  Retrying incomplete response with commits-per-request " + dynamicCPR[0]); //$NON-NLS-1$
continue;
}
i = end;
if (llmClient.wasLastResponseTruncated() && dynamicCPR[0] > 1) {
dynamicCPR[0] = Math.max(1, dynamicCPR[0] / 2);
miningLog.println("  Reducing commits-per-request to " + dynamicCPR[0] + " after truncated response"); //$NON-NLS-1$
}
if (llmClient.isApiUnavailable()) {
logApiUnavailable(llmClient);
throw new IOException("LLM API unavailable; stopping mining"); //$NON-NLS-1$
}
}
batch = walker.nextBatch(lastBatchCommit,
effectiveStartDate, effectiveEndDate, batchSize);
}

// Epoch rotation: if epochs are configured and current epoch is exhausted,
// advance to the next epoch and reset lastProcessedCommit
if (!epochs.isEmpty()) {
int epochIdx = repoState.getCurrentEpoch();
MiningConfig.EpochEntry completedEpoch = epochs.get(epochIdx);
repoState.getCompletedEpochs().add(completedEpoch.toString());
repoState.setCurrentEpoch(epochIdx + 1);
repoState.setLastProcessedCommit(null);
repoState.setStatus("EPOCH_COMPLETE"); //$NON-NLS-1$
miningLog.println("  Epoch " + epochIdx + " complete (" + completedEpoch //$NON-NLS-1$ //$NON-NLS-2$
+ "). Advancing to epoch " + (epochIdx + 1)); //$NON-NLS-1$
state.save(statePath);
}
}

miningLog.println("  Completed: " + repo.getUrl()); //$NON-NLS-1$
}
}

private void processBatch(List<RevCommit> commits, RepoEntry repo,
DiffExtractor diffExtractor, MiningState state, Path statePath,
LlmClient llmClient, PromptBuilder promptBuilder,
String dslContext, CategoryManager categoryManager, DslValidator validator,
StatisticsCollector stats, ReportAggregator aggregator,
int minDiffLines, int maxDiffLines,
TypeContextEnricher typeEnricher,
CommitKeywordFilter keywordFilter, Path outputDir, Path sandboxRoot) throws IOException {
// Classify commits in original order: track which are skipped vs included
List<CommitData> commitDataList = new ArrayList<>();
List<Boolean> isSkipped = new ArrayList<>();
List<Integer> diffLineCounts = new ArrayList<>();

RepoState repoState = state.getRepoState(repo.getUrl());
int effectiveMaxDiff = repoState.getLearnedMaxDiffLines() > 0
? repoState.getLearnedMaxDiffLines() : maxDiffLines;

for (RevCommit commit : commits) {
CommitEvaluation completed = completedEvaluations.getOrDefault(repo.getUrl(), Map.of()).get(commit.getName());
if (completed != null) {
// A previous partial batch may have completed this commit beyond a gap.
// Reuse its exact result without another API call or duplicate statistics.
reusedEvaluations.add(completed);
isSkipped.add(Boolean.TRUE);
repoState.removeDeferredCommit(commit.getName());
miningLog.println("  Reusing completed evaluation for " + formatCommitInfo(commit, repo)); //$NON-NLS-1$
continue;
}
// Skip commits that don't match keyword filter (if active)
if (keywordFilter != null && !keywordFilter.matches(commit.getFullMessage())) {
miningLog.println("  Skipping commit " + formatCommitInfo(commit, repo) //$NON-NLS-1$
+ " (no keyword match)"); //$NON-NLS-1$
isSkipped.add(Boolean.TRUE);
continue;
}
String diff = diffExtractor.extractDiff(commit);
int lineCount = diff.split("\n", -1).length; //$NON-NLS-1$
if (diff.isBlank()) {
isSkipped.add(Boolean.TRUE);
} else if (lineCount < minDiffLines) {
miningLog.println("  Skipping commit " + formatCommitInfo(commit, repo) //$NON-NLS-1$
+ " (diff too small: " + lineCount + " lines)"); //$NON-NLS-1$ //$NON-NLS-2$
isSkipped.add(Boolean.TRUE);
} else if (lineCount > effectiveMaxDiff) {
miningLog.println("  Deferring commit " + formatCommitInfo(commit, repo) //$NON-NLS-1$
+ " (" + lineCount + " lines > limit " + effectiveMaxDiff + ")"); //$NON-NLS-1$ //$NON-NLS-2$
String shortMsg = commit.getShortMessage();
String truncatedMsg = shortMsg == null
? "" //$NON-NLS-1$
: shortMsg.substring(0, Math.min(120, shortMsg.length()));
repoState.addDeferredCommit(new DeferredCommit(
commit.getName(),
truncatedMsg,
lineCount, "DIFF_TOO_LARGE", Instant.now().toString(), 0, 3)); //$NON-NLS-1$
isSkipped.add(Boolean.TRUE);
} else {
commitDataList.add(new CommitData(commit.getName(), commit.getFullMessage(), diff));
diffLineCounts.add(lineCount);
isSkipped.add(Boolean.FALSE);
}
}

List<CommitEvaluation> evaluations = null;
IOException evaluationFailure = null;
Map<String, CommitEvaluation> evaluationsByHash = new HashMap<>();
if (!commitDataList.isEmpty()) {
// Enrich prompt with Eclipse type context from actual diffs if enabled
if (typeEnricher != null) {
// Clear stale type context from previous batch before enriching
promptBuilder.setTypeContext(null);
StringBuilder combinedDiffs = new StringBuilder();
for (CommitData cd : commitDataList) {
combinedDiffs.append(cd.diff()).append('\n');
}
String context = typeEnricher.enrichFromDiff(combinedDiffs.toString());
if (!context.isEmpty()) {
promptBuilder.setTypeContext(context);
}
}
List<String> hashes = commitDataList.stream().map(CommitData::commitHash).toList();
List<String> messages = commitDataList.stream().map(CommitData::commitMessage).toList();
String previousResults = knownRulesContext.isEmpty() ? null : knownRulesContext;
String prompt = promptBuilder.buildBatchPrompt(dslContext,
categoryManager.getCategoriesJson(), commitDataList, previousResults);
try {
evaluations = llmClient.evaluateBatch(prompt, hashes, messages, repo.getUrl());
} catch (IOException e) {
evaluationFailure = e;
}
if (evaluations != null) {
for (CommitEvaluation evaluation : evaluations) {
if (evaluation != null) {
evaluationsByHash.putIfAbsent(evaluation.commitHash(), evaluation);
}
}
}

// Do not lower the limit for an incomplete batch: its unresolved commits must
// still reach the API when the next run resumes at the gap.
if (evaluationFailure == null && llmClient.wasLastResponseTruncated() && !diffLineCounts.isEmpty()
&& commitDataList.stream().allMatch(commit -> evaluationsByHash.containsKey(commit.commitHash()))) {
int maxDiffInBatch = diffLineCounts.stream().mapToInt(Integer::intValue).max().orElse(0);
if (maxDiffInBatch > 0) {
int newLimit = (int) (maxDiffInBatch * 0.8);
if (repoState.getLearnedMaxDiffLines() == -1 || newLimit < repoState.getLearnedMaxDiffLines()) {
miningLog.println("  Learning: reducing max diff limit to " + newLimit //$NON-NLS-1$
+ " lines (was " + (repoState.getLearnedMaxDiffLines() == -1 ? "default" : repoState.getLearnedMaxDiffLines()) + ")"); //$NON-NLS-1$ //$NON-NLS-2$
repoState.setLearnedMaxDiffLines(newLimit);
}
}
}

}

// A malformed response element can leave a gap in the returned list. Match by
// commit identity, retain every actual result, and advance only its complete prefix.
if (evaluations != null) {
miningLog.println("  Received " + evaluationsByHash.size() + " evaluations for " //$NON-NLS-1$ //$NON-NLS-2$
+ repo.getUrl() + " [" + repo.getBranch() + "]:"); //$NON-NLS-1$ //$NON-NLS-2$
}
boolean completePrefix = true;
int firstMissingIndex = -1;
int diffIdx = 0;
for (int i = 0; i < commits.size(); i++) {
RevCommit commit = commits.get(i);
if (isSkipped.get(i)) {
if (completePrefix) {
state.updateLastProcessedCommit(repo.getUrl(), commit.getName());
CommitEvaluation reused = completedEvaluations.getOrDefault(repo.getUrl(), Map.of()).get(commit.getName());
if (reused != null && reused.relevant() && reused.category() != null) {
repoState.incrementCategoryHitCount(reused.category());
}
}
} else {
int diffLines = diffLineCounts.get(diffIdx++);
CommitEvaluation evaluation = evaluationsByHash.get(commit.getName());
if (evaluation == null) {
if (completePrefix) {
firstMissingIndex = i;
}
completePrefix = false;
String message = commit.getShortMessage();
repoState.addDeferredCommit(new DeferredCommit(commit.getName(),
message == null ? "" : message.substring(0, Math.min(120, message.length())), //$NON-NLS-1$
diffLines, evaluationFailure == null ? "INCOMPLETE_BATCH" : "API_FAILURE", //$NON-NLS-1$ //$NON-NLS-2$
Instant.now().toString(), 0, 3));
miningLog.println("  Missing evaluation for commit " + formatCommitInfo(commit, repo) //$NON-NLS-1$
+ "; retaining the checkpoint before this commit for retry."); //$NON-NLS-1$
continue;
}
String diffInfo = " [" + diffLines + " lines]"; //$NON-NLS-1$ //$NON-NLS-2$
miningLog.println("    - " + formatCommitInfo(commit, repo) + diffInfo + " -> " + evaluation.trafficLight()); //$NON-NLS-1$ //$NON-NLS-2$
handleEvaluation(evaluation, commit, repo, validator, categoryManager, stats, aggregator);
if (completePrefix) {
state.updateLastProcessedCommit(repo.getUrl(), commit.getName());
// Persist hit counts only for the complete prefix, including cached results
// when a later run reaches them. This avoids recounting results after a gap.
if (evaluation.relevant() && evaluation.category() != null) {
repoState.incrementCategoryHitCount(evaluation.category());
}
}
// Remove from deferred list if it was previously deferred
repoState.removeDeferredCommit(commit.getName());
}
}

// A persisted scan position must never get ahead of its candidate data.
try {
saveCheckpoint(aggregator, validator, state, statePath, outputDir, sandboxRoot);
} catch (IOException e) {
if (evaluationFailure != null) {
evaluationFailure.addSuppressed(e);
throw evaluationFailure;
}
throw e;
}
if (evaluationFailure != null) {
throw evaluationFailure;
}
if (!completePrefix) {
throw new IncompleteBatchException(repo.getUrl(), firstMissingIndex);
}
}

private static final class IncompleteBatchException extends IOException {
private static final long serialVersionUID = 1L;
private final int firstMissingIndex;

IncompleteBatchException(String repoUrl, int firstMissingIndex) {
super("Incomplete batch evaluation for " + repoUrl //$NON-NLS-1$
+ "; completed results saved, unevaluated commits remain for retry"); //$NON-NLS-1$
this.firstMissingIndex = firstMissingIndex;
}
}

private void saveCheckpoint(ReportAggregator aggregator, DslValidator validator,
MiningState state, Path statePath, Path outputDir, Path sandboxRoot) throws IOException {
new JsonReporter().writeEvaluations(aggregator.getAllEvaluations(), outputDir);
saveRunCandidates(aggregator, validator, sandboxRoot);
state.save(statePath);
}

private void saveRunCandidates(ReportAggregator aggregator, DslValidator validator, Path sandboxRoot) throws IOException {
CandidateStore candidateStore = new CandidateStore(sandboxRoot.resolve("mining-candidates")); //$NON-NLS-1$
stagedCandidateCount += saveCandidates(aggregator.getAllEvaluations(), candidateStore, validator, miningLog).size();
stagedCandidateCount += saveCandidates(reusedEvaluations, candidateStore, validator, miningLog, true).size();
}

private void handleEvaluation(CommitEvaluation evaluation, RevCommit commit, RepoEntry repo,
DslValidator validator, CategoryManager categoryManager,
StatisticsCollector stats, ReportAggregator aggregator) {
String validationResult = null;
if (evaluation.dslRule() != null && !evaluation.dslRule().isBlank()) {
var validation = validator.validate(evaluation.dslRule());
if (validation.valid()) {
validationResult = "VALID"; //$NON-NLS-1$
} else {
validationResult = validation.message();
miningLog.println("  Invalid DSL rule for " + formatCommitInfo(commit, repo) //$NON-NLS-1$
+ ": " + validation.message()); //$NON-NLS-1$
if (Boolean.parseBoolean(System.getenv("GEMINI_DEBUG"))) { //$NON-NLS-1$
miningLog.println("  --- DSL rule begin ---"); //$NON-NLS-1$
miningLog.println(evaluation.dslRule());
miningLog.println("  --- DSL rule end ---"); //$NON-NLS-1$
}
}
}
// Create evaluation with validation result and commit date (author date)
Instant commitDate = commit.getAuthorIdent().getWhen().toInstant();
CommitEvaluation enriched = new CommitEvaluation(
evaluation.commitHash(), evaluation.commitMessage(), evaluation.repoUrl(),
evaluation.evaluatedAt(), commitDate, evaluation.relevant(), evaluation.irrelevantReason(),
evaluation.isDuplicate(), evaluation.duplicateOf(),
evaluation.reusability(), evaluation.codeImprovement(), evaluation.implementationEffort(),
evaluation.trafficLight(), evaluation.category(), evaluation.isNewCategory(),
evaluation.categoryReason(), evaluation.canImplementInCurrentDsl(),
evaluation.dslRule(), evaluation.targetHintFile(),
evaluation.languageChangeNeeded(), evaluation.dslRuleAfterChange(),
evaluation.summary(), validationResult,
evaluation.beforeExample(), evaluation.afterExample(), evaluation.negativeExample());
if (evaluation.isNewCategory() && evaluation.category() != null) {
categoryManager.addCategory(evaluation.category());
}
stats.record(enriched);
aggregator.add(enriched);
completedEvaluations.computeIfAbsent(repo.getUrl(), key -> new HashMap<>()).put(commit.getName(), enriched);
}

static String repoDirectoryName(String url) {
String name = url;
if (name.endsWith(".git")) {
name = name.substring(0, name.length() - 4);
}
int lastSlash = name.lastIndexOf('/');
if (lastSlash >= 0) {
name = name.substring(lastSlash + 1);
}
return name;
}

static String formatCommitInfo(RevCommit commit, RepoEntry repo) {
String datetime = COMMIT_DATE_FORMAT.format(commit.getAuthorIdent().getWhen().toInstant());
String title = commit.getShortMessage().replace("\"", "\\\""); //$NON-NLS-1$ //$NON-NLS-2$
return commit.getName().substring(0, 7) + " on " + repo.getBranch() //$NON-NLS-1$
+ " (" + datetime + ") \"" + title + "\""; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}

private void logApiUnavailable(LlmClient llmClient) {
miningLog.println("LLM API has been unreachable for over " //$NON-NLS-1$
+ llmClient.getMaxFailureDuration().toMinutes()
+ " minutes. Stopping to avoid wasting CI time. State saved; will resume on next run."); //$NON-NLS-1$
}

private static void printUsage() {
System.out.println("Usage: java -jar sandbox-mining-core.jar [options]"); //$NON-NLS-1$
System.out.println("Options:"); //$NON-NLS-1$
System.out.println("  --config <path>              Path to repos.yml config file"); //$NON-NLS-1$
System.out.println("  --state <path>               Path to state.json file"); //$NON-NLS-1$
System.out.println("  --sandbox-root <path>        Root of sandbox repository"); //$NON-NLS-1$
System.out.println("  --batch-size <n>             Number of commits per batch (default: 500)"); //$NON-NLS-1$
System.out.println("  --commits-per-request <n>    Commits grouped into one API call (default: 4)"); //$NON-NLS-1$
System.out.println("  --output <path>              Output directory (default: output)"); //$NON-NLS-1$
System.out.println("  --max-failure-duration <s>   Seconds without a successful API call before aborting (default: 300)"); //$NON-NLS-1$
System.out.println("  --llm-provider <name>        LLM provider: gemini, openai, deepseek, qwen, llama, or mistral (default: auto-detect)"); //$NON-NLS-1$
System.out.println("  --retry-deferred             Retry previously deferred commits"); //$NON-NLS-1$
System.out.println("  --reset-learned-limits       Reset learned diff size limits"); //$NON-NLS-1$
System.out.println("  --commit-list <path>         Path to file with commit hashes to process (one per line)"); //$NON-NLS-1$
System.out.println("  --max-duration <minutes>     Maximum run duration in minutes"); //$NON-NLS-1$
System.out.println("  --comparison-mode            Enable comparison against reference results"); //$NON-NLS-1$
System.out.println("  --copilot-results <path>     Path to Copilot/reference evaluation results JSON"); //$NON-NLS-1$
System.out.println("  --enrich-type-context        Add Eclipse type hierarchy context to prompts"); //$NON-NLS-1$
System.out.println("  --keyword-filter <path>      Path to keyword filter file for commit pre-filtering"); //$NON-NLS-1$
System.out.println("  --output-format <format>     Output format: json, netbeans, or both (default: json)"); //$NON-NLS-1$
System.out.println("  --strict-netbeans            Only NetBeans format on stdout, info on stderr"); //$NON-NLS-1$
}

private static String requireArg(String[] args, int index, String option) {
if (index >= args.length) {
throw new IllegalArgumentException("Option " + option + " requires a value");
}
return args[index];
}

/**
 * Creates {@link MiningCandidate} objects from GREEN+VALID evaluations
 * and saves them to the candidate store. Only evaluations with GREEN traffic
 * light and VALID DSL validation result are staged as candidates.
 *
 * @param evaluations    all evaluations from the current mining run
 * @param candidateStore the store to save candidates to
 * @param validator      the DSL validator (for final validation check)
 * @param log            the log stream for informational output
 * @return the list of newly saved candidates
 */
static List<MiningCandidate> saveCandidates(List<CommitEvaluation> evaluations,
		CandidateStore candidateStore, DslValidator validator, PrintStream log) throws IOException {
	return saveCandidates(evaluations, candidateStore, validator, log, false);
}

private static List<MiningCandidate> saveCandidates(List<CommitEvaluation> evaluations,
		CandidateStore candidateStore, DslValidator validator, PrintStream log, boolean preserveExisting) throws IOException {
	List<MiningCandidate> saved = new ArrayList<>();
	for (CommitEvaluation eval : evaluations) {
		if (eval.trafficLight() != CommitEvaluation.TrafficLight.GREEN) {
			continue;
		}
		if (!"VALID".equals(eval.dslValidationResult())) { //$NON-NLS-1$
			continue;
		}
		String dslRule = eval.dslRule();
		if (dslRule == null || dslRule.isBlank()) {
			continue;
		}
		var validation = validator.validate(dslRule);
		if (!validation.valid()) {
			log.println("  Skipped candidate due to invalid DSL during final validation: " //$NON-NLS-1$
					+ eval.commitHash());
			continue;
		}
		MiningCandidate candidate = new MiningCandidate(
				dslRule,
				eval.beforeExample(),
				eval.afterExample(),
				eval.negativeExample(),
				eval.targetHintFile(),
				eval.commitHash(),
				eval.repoUrl(),
				eval.category(),
				eval.summary(),
				Instant.now().toString());
		// Replaying saved evaluations must not replace a reviewed candidate's
		// corrected content or revision, but still recreate a missing candidate.
		if (preserveExisting ? candidateStore.findById(candidate.getCandidateId()).isPresent()
				: candidateStore.containsCandidate(candidate)) {
			continue;
		}
		candidate.setStatus(CandidateStatus.DSL_VALID);
		candidateStore.save(candidate);
		saved.add(candidate);
		log.println("  Staged candidate: " + candidate.toFileName()); //$NON-NLS-1$
	}
	if (!saved.isEmpty()) {
		log.println("Staged " + saved.size() + " new candidates in mining-candidates/"); //$NON-NLS-1$ //$NON-NLS-2$
	}
	return saved;
}

private static void deleteDirectory(Path dir) throws IOException {
if (Files.exists(dir)) {
try (var walk = Files.walk(dir)) {
walk.sorted(java.util.Comparator.reverseOrder())
.forEach(p -> {
try {
Files.deleteIfExists(p);
} catch (IOException e) {
// best effort cleanup
}
});
}
}
}

private void retryDeferredCommits(MiningState state, Path statePath, Path workDir,
LlmClient llmClient, PromptBuilder promptBuilder, String dslContext,
CategoryManager categoryManager, DslValidator validator,
StatisticsCollector stats, ReportAggregator aggregator,
MiningConfig config, boolean forceRetry) throws IOException {
for (RepoEntry repo : config.getRepositories()) {
RepoState repoState = state.getRepoState(repo.getUrl());
List<DeferredCommit> toRetry = new ArrayList<>(repoState.getDeferredCommits());
if (toRetry.isEmpty()) continue;

miningLog.println("Processing " + toRetry.size() + " deferred commits for " + repo.getUrl()); //$NON-NLS-1$ //$NON-NLS-2$
List<DeferredCommit> remaining = new ArrayList<>();
for (DeferredCommit dc : toRetry) {
if (!forceRetry && dc.getRetryCount() >= dc.getMaxRetries()) {
repoState.moveToPermanentlySkipped(dc.getHash());
continue;
}
// No actual retry can be performed here because the repository checkout and
// diff extraction pipeline are not available in this context. Keep the commit
// deferred with its current retryCount so it can be retried during the next
// normal processing run when its diff is available again.
remaining.add(dc);
}
repoState.setDeferredCommits(remaining);
state.save(statePath);
}
}

private void printRunSummary(StatisticsCollector stats, MiningState state,
MiningConfig config, LlmClient llmClient, long startTimeMs) {
long durationMs = System.currentTimeMillis() - startTimeMs;
long minutes = durationMs / 60000;
long seconds = (durationMs % 60000) / 1000;
int totalDeferred = 0;
int totalPermanentlySkipped = 0;
for (RepoEntry repo : config.getRepositories()) {
RepoState rs = state.getRepoState(repo.getUrl());
totalDeferred += rs.getDeferredCommits().size();
totalPermanentlySkipped += rs.getPermanentlySkipped().size();
}
miningLog.println("=== Mining Run Summary ==="); //$NON-NLS-1$
miningLog.println("Duration:           " + minutes + "m " + seconds + "s"); //$NON-NLS-1$ //$NON-NLS-2$
miningLog.println("Commits processed:  " + stats.getTotalProcessed()); //$NON-NLS-1$
miningLog.println("Commits deferred:   " + totalDeferred); //$NON-NLS-1$
miningLog.println("Commits permanently skipped: " + totalPermanentlySkipped); //$NON-NLS-1$
miningLog.println("Relevant:           " + stats.getRelevant()); //$NON-NLS-1$
miningLog.println("API calls:          " + llmClient.getDailyRequestCount()); //$NON-NLS-1$
miningLog.println("Model:              " + llmClient.getModel()); //$NON-NLS-1$
// Learned limits
for (RepoEntry repo : config.getRepositories()) {
RepoState rs = state.getRepoState(repo.getUrl());
if (rs.getLearnedMaxDiffLines() > 0) {
miningLog.println("Learned max diff:   " + rs.getLearnedMaxDiffLines() + " lines (for " + repo.getUrl() + ")"); //$NON-NLS-1$ //$NON-NLS-2$
}
}
}

private void printDeferredReport(MiningState state, MiningConfig config) {
boolean hasDeferred = false;
for (RepoEntry repo : config.getRepositories()) {
RepoState rs = state.getRepoState(repo.getUrl());
if (!rs.getDeferredCommits().isEmpty()) {
if (!hasDeferred) {
miningLog.println("Deferred commits for retry:"); //$NON-NLS-1$
hasDeferred = true;
}
miningLog.println("  " + repo.getUrl() + ": " + rs.getDeferredCommits().size() + " commits"); //$NON-NLS-1$ //$NON-NLS-2$
for (DeferredCommit dc : rs.getDeferredCommits()) {
miningLog.println("    - " + dc.getHash().substring(0, Math.min(7, dc.getHash().length())) //$NON-NLS-1$
+ " (" + dc.getDiffLines() + " lines, " + dc.getReason() //$NON-NLS-1$ //$NON-NLS-2$
+ ", retry " + dc.getRetryCount() + "/" + dc.getMaxRetries() + ")"); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}
}
}
}

/**
 * Checks whether the mining run should stop based on elapsed time.
 *
 * @param startTimeMs     the start time in milliseconds
 * @param maxDurationMin  maximum duration in minutes (0 = no limit)
 * @return true if the run should stop
 */
static boolean shouldStop(long startTimeMs, int maxDurationMin) {
if (maxDurationMin <= 0) {
return false;
}
long elapsed = System.currentTimeMillis() - startTimeMs;
return elapsed >= (long) maxDurationMin * 60 * 1000;
}

/**
 * Reads commit hashes from a file (one per line).
 * Blank lines and lines starting with {@code #} are ignored.
 *
 * @param commitListPath path to the commit list file
 * @return list of commit hashes
 * @throws IOException if the file cannot be read
 */
static List<String> readCommitList(Path commitListPath) throws IOException {
List<String> hashes = new ArrayList<>();
for (String line : Files.readAllLines(commitListPath, StandardCharsets.UTF_8)) {
String trimmed = line.trim();
if (!trimmed.isEmpty() && !trimmed.startsWith("#")) { //$NON-NLS-1$
hashes.add(trimmed);
}
}
return hashes;
}
}