Skip to content

Commit 9114bba

Browse files
vladakvladak
authored
History cache control per project (#4748)
* allow to enable/disable history cache per project fixes #4745
1 parent 7b5f14f commit 9114bba

File tree

7 files changed

+246
-53
lines changed

7 files changed

+246
-53
lines changed

‎opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Project.java

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,15 @@ public class Project implements Comparable<Project>, Nameable, Serializable {
8484
private Boolean handleRenamedFiles = null;
8585

8686
/**
87-
* This flag enables/disables per-project history cache.
87+
* This flag enables/disables per-project history queries.
8888
*/
8989
private Boolean historyEnabled = null;
9090

91+
/**
92+
* This flag enables/disables per-project history cache.
93+
*/
94+
private Boolean historyCacheEnabled = null;
95+
9196
/**
9297
* This flag enables/disables per-project annotation cache.
9398
*/
@@ -301,6 +306,20 @@ public void setHistoryEnabled(boolean flag) {
301306
this.historyEnabled = flag;
302307
}
303308

309+
/**
310+
* @return true if this project should have history cache.
311+
*/
312+
public boolean isHistoryCacheEnabled() {
313+
return historyCacheEnabled != null && historyCacheEnabled;
314+
}
315+
316+
/**
317+
* @param flag true if project should have history cache, false otherwise.
318+
*/
319+
public void setHistoryCacheEnabled(boolean flag) {
320+
this.historyCacheEnabled = flag;
321+
}
322+
304323
/**
305324
* @return true if this project should have annotation cache.
306325
*/
@@ -494,11 +513,16 @@ public final void completeWithDefaults() {
494513
setHandleRenamedFiles(env.isHandleHistoryOfRenamedFiles());
495514
}
496515

497-
// Allow project to override global setting of history cache generation.
516+
// Allow project to override global setting of history queries.
498517
if (historyEnabled == null) {
499518
setHistoryEnabled(env.isHistoryEnabled());
500519
}
501520

521+
// Allow project to override global setting of history cache generation.
522+
if (historyCacheEnabled == null) {
523+
setHistoryCacheEnabled(env.useHistoryCache());
524+
}
525+
502526
// Allow project to override global setting of annotation cache generation.
503527
if (annotationCacheEnabled == null) {
504528
setAnnotationCacheEnabled(env.isAnnotationCacheEnabled());

‎opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryGuru.java

Lines changed: 63 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,11 @@ private HistoryGuru() {
135135
repositoryLookup = RepositoryLookup.cached();
136136
}
137137

138+
@VisibleForTesting
139+
HistoryCache getHistoryCache() {
140+
return historyCache;
141+
}
142+
138143
/**
139144
* Set annotation cache to its default implementation.
140145
* @return {@link AnnotationCache} instance or {@code null} on error
@@ -164,18 +169,16 @@ static AnnotationCache initializeAnnotationCache() {
164169
* @return {@link HistoryCache} instance
165170
*/
166171
private HistoryCache initializeHistoryCache() {
167-
HistoryCache historyCacheResult = null;
168-
if (env.useHistoryCache()) {
169-
historyCacheResult = new FileHistoryCache();
172+
HistoryCache historyCacheResult = new FileHistoryCache();
170173

171-
try {
172-
historyCacheResult.initialize();
173-
} catch (CacheException he) {
174-
LOGGER.log(Level.WARNING, "Failed to initialize the history cache", he);
175-
// Failed to initialize, run without a history cache.
176-
historyCacheResult = null;
177-
}
174+
try {
175+
historyCacheResult.initialize();
176+
} catch (CacheException he) {
177+
LOGGER.log(Level.WARNING, "Failed to initialize the history cache", he);
178+
// Failed to initialize, run without a history cache.
179+
historyCacheResult = null;
178180
}
181+
179182
return historyCacheResult;
180183
}
181184

@@ -188,13 +191,24 @@ public static HistoryGuru getInstance() {
188191
return INSTANCE;
189192
}
190193

191-
/**
192-
* Return whether cache should be used for the history log.
193-
*
194-
* @return {@code true} if the history cache has been enabled and initialized, {@code false} otherwise
195-
*/
196-
private boolean useHistoryCache() {
197-
return historyCache != null;
194+
private boolean useHistoryCache(File file) {
195+
if (historyCache == null) {
196+
return false;
197+
}
198+
199+
return useHistoryCache(getRepository(file));
200+
}
201+
202+
private boolean useHistoryCache(@Nullable Repository repository) {
203+
if (historyCache == null || repository == null) {
204+
return false;
205+
}
206+
207+
if (!historyCache.supportsRepository(repository)) {
208+
return false;
209+
}
210+
211+
return repository.isHistoryCacheEnabled();
198212
}
199213

200214
/**
@@ -418,7 +432,7 @@ boolean isRepoHistoryEligible(Repository repo, File file, boolean ui) {
418432
private History getHistoryFromCache(File file, Repository repository, boolean withFiles)
419433
throws CacheException {
420434

421-
if (useHistoryCache() && historyCache.supportsRepository(repository)) {
435+
if (useHistoryCache(repository)) {
422436
return historyCache.get(file, repository, withFiles);
423437
}
424438

@@ -428,7 +442,7 @@ private History getHistoryFromCache(File file, Repository repository, boolean wi
428442
@Nullable
429443
private HistoryEntry getLastHistoryEntryFromCache(File file, Repository repository) throws CacheException {
430444

431-
if (useHistoryCache() && historyCache.supportsRepository(repository)) {
445+
if (useHistoryCache(repository)) {
432446
return historyCache.getLastHistoryEntry(file);
433447
}
434448

@@ -451,6 +465,9 @@ public HistoryEntry getLastHistoryEntry(File file, boolean ui, boolean fallback)
451465
launderLog(file.toString())));
452466
final File dir = file.isDirectory() ? file : file.getParentFile();
453467
final Repository repository = getRepository(dir);
468+
if (repository == null) {
469+
return null;
470+
}
454471
final String meterName = "history.entry.latest";
455472

456473
try {
@@ -479,7 +496,7 @@ public HistoryEntry getLastHistoryEntry(File file, boolean ui, boolean fallback)
479496
if (!isRepoHistoryEligible(repository, file, ui)) {
480497
statistics.report(LOGGER, Level.FINEST,
481498
String.format("cannot retrieve the last history entry for ''%s'' in %s because of settings",
482-
launderLog(file.toString()), repository), meterName);
499+
launderLog(file.toString()), repository), meterName);
483500
return null;
484501
}
485502

@@ -691,7 +708,7 @@ private static boolean repositoryHasHistory(File file, Repository repo) {
691708
* @return if there is history cache entry for the file
692709
*/
693710
public boolean hasHistoryCacheForFile(File file) {
694-
if (!useHistoryCache()) {
711+
if (!useHistoryCache(file)) {
695712
LOGGER.finest(() -> String.format("history cache is off for '%s' to check history cache presence",
696713
launderLog(file.toString())));
697714
return false;
@@ -823,15 +840,15 @@ public boolean fillLastHistoryEntries(File directory, List<DirectoryEntry> entri
823840
return true;
824841
}
825842

826-
if (!useHistoryCache()) {
827-
LOGGER.finest(() -> String.format("history cache is disabled for '%s' to retrieve last modified times",
843+
Repository repository = getRepository(directory);
844+
if (repository == null) {
845+
LOGGER.finest(() -> String.format("cannot find repository for '%s' to retrieve last modified times",
828846
launderLog(directory.toString())));
829847
return true;
830848
}
831849

832-
Repository repository = getRepository(directory);
833-
if (repository == null) {
834-
LOGGER.finest(() -> String.format("cannot find repository for '%s' to retrieve last modified times",
850+
if (!useHistoryCache(repository)) {
851+
LOGGER.finest(() -> String.format("history cache is disabled for '%s' to retrieve last modified times",
835852
launderLog(directory.toString())));
836853
return true;
837854
}
@@ -1036,9 +1053,17 @@ public void storeHistory(File file, History history) {
10361053
}
10371054

10381055
private void createHistoryCache(Repository repository, String sinceRevision) throws CacheException, HistoryException {
1056+
if (!repository.isHistoryCacheEnabled()) {
1057+
LOGGER.log(Level.INFO,
1058+
"Skipping history cache creation for {0} and its subdirectories: history cache disabled",
1059+
repository);
1060+
return;
1061+
}
1062+
10391063
if (!repository.isHistoryEnabled()) {
10401064
LOGGER.log(Level.INFO,
1041-
"Skipping history cache creation for {0} and its subdirectories", repository);
1065+
"Skipping history cache creation for {0} and its subdirectories: history disabled",
1066+
repository);
10421067
return;
10431068
}
10441069

@@ -1139,9 +1164,14 @@ private Map<Repository, Optional<Exception>> createHistoryCacheReal(Collection<R
11391164
* @return map of repository to optional exception
11401165
*/
11411166
public Map<Repository, Optional<Exception>> createHistoryCache(Collection<String> repositories) {
1142-
if (!useHistoryCache()) {
1167+
if (repositories.stream().
1168+
map(e -> new File(env.getSourceRootPath(), e)).
1169+
map(this::getRepository).
1170+
filter(Objects::nonNull).
1171+
noneMatch(RepositoryInfo::isHistoryCacheEnabled)) {
11431172
return Collections.emptyMap();
11441173
}
1174+
11451175
return createHistoryCacheReal(getReposFromString(repositories));
11461176
}
11471177

@@ -1151,12 +1181,12 @@ public Map<Repository, Optional<Exception>> createHistoryCache(Collection<String
11511181
* @param removeHistory whether to remove history cache entry for the path
11521182
*/
11531183
public void clearHistoryCacheFile(String path, boolean removeHistory) {
1154-
if (!useHistoryCache()) {
1184+
Repository repository = getRepository(new File(env.getSourceRootFile(), path));
1185+
if (repository == null) {
11551186
return;
11561187
}
11571188

1158-
Repository repository = getRepository(new File(env.getSourceRootFile(), path));
1159-
if (repository == null) {
1189+
if (!useHistoryCache(repository)) {
11601190
return;
11611191
}
11621192

@@ -1230,7 +1260,7 @@ public void clearAnnotationCacheFile(String path) {
12301260
* @return list of repository names
12311261
*/
12321262
public List<String> removeHistoryCache(Collection<RepositoryInfo> repositories) {
1233-
if (!useHistoryCache()) {
1263+
if (repositories.stream().noneMatch(RepositoryInfo::isHistoryCacheEnabled)) {
12341264
return List.of();
12351265
}
12361266

@@ -1258,7 +1288,7 @@ public List<String> removeAnnotationCache(Collection<RepositoryInfo> repositorie
12581288
* @return map of repository to optional exception
12591289
*/
12601290
public Map<Repository, Optional<Exception>> createHistoryCache() {
1261-
if (!useHistoryCache()) {
1291+
if (repositories.values().stream().noneMatch(RepositoryInfo::isHistoryCacheEnabled)) {
12621292
return Collections.emptyMap();
12631293
}
12641294

‎opengrok-indexer/src/main/java/org/opengrok/indexer/history/RepositoryInfo.java

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@ public class RepositoryInfo implements Serializable {
7878
@DTOElement
7979
private boolean historyEnabled;
8080
@DTOElement
81+
private boolean historyCacheEnabled;
82+
@DTOElement
8183
private boolean annotationCacheEnabled;
8284
@DTOElement
8385
private boolean mergeCommitsEnabled;
@@ -105,6 +107,7 @@ public RepositoryInfo(RepositoryInfo orig) {
105107
this.branch = orig.branch;
106108
this.currentVersion = orig.currentVersion;
107109
this.historyEnabled = orig.historyEnabled;
110+
this.historyCacheEnabled = orig.historyCacheEnabled;
108111
this.annotationCacheEnabled = orig.annotationCacheEnabled;
109112
this.handleRenamedFiles = orig.handleRenamedFiles;
110113
this.mergeCommitsEnabled = orig.mergeCommitsEnabled;
@@ -164,7 +167,7 @@ public void setHistoryBasedReindex(boolean flag) {
164167
}
165168

166169
/**
167-
* @return true if the repository should have history cache.
170+
* @return true if the repository should support history queries.
168171
*/
169172
public boolean isHistoryEnabled() {
170173
return this.historyEnabled;
@@ -174,6 +177,17 @@ public void setHistoryEnabled(boolean flag) {
174177
this.historyEnabled = flag;
175178
}
176179

180+
/**
181+
* @return true if the history for this repository should be stored in history cache.
182+
*/
183+
public boolean isHistoryCacheEnabled() {
184+
return this.historyCacheEnabled;
185+
}
186+
187+
public void setHistoryCacheEnabled(boolean flag) {
188+
this.historyCacheEnabled = flag;
189+
}
190+
177191
public boolean isAnnotationCacheEnabled() {
178192
return annotationCacheEnabled;
179193
}
@@ -381,6 +395,7 @@ public void fillFromProject() {
381395
Project proj = Project.getProject(getDirectoryNameRelative());
382396
if (proj != null) {
383397
setHistoryEnabled(proj.isHistoryEnabled());
398+
setHistoryCacheEnabled(proj.isHistoryCacheEnabled());
384399
setAnnotationCacheEnabled(proj.isAnnotationCacheEnabled());
385400
setHandleRenamedFiles(proj.isHandleRenamedFiles());
386401
setMergeCommitsEnabled(proj.isMergeCommitsEnabled());
@@ -391,6 +406,7 @@ public void fillFromProject() {
391406
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
392407

393408
setHistoryEnabled(env.isHistoryEnabled());
409+
setHistoryCacheEnabled(env.useHistoryCache());
394410
setAnnotationCacheEnabled(env.isAnnotationCacheEnabled());
395411
setHandleRenamedFiles(env.isHandleHistoryOfRenamedFiles());
396412
setMergeCommitsEnabled(env.isMergeCommitsEnabled());
@@ -433,9 +449,20 @@ public String toString() {
433449
stringBuilder.append(",");
434450

435451
if (!isHistoryEnabled()) {
452+
stringBuilder.append("history=off");
453+
} else {
454+
stringBuilder.append("history=on");
455+
}
456+
457+
stringBuilder.append(",");
458+
if (!isHistoryCacheEnabled()) {
436459
stringBuilder.append("historyCache=off");
437460
} else {
438461
stringBuilder.append("historyCache=on,");
462+
}
463+
464+
if (isHandleRenamedFiles()) {
465+
stringBuilder.append(",");
439466
stringBuilder.append("renamed=");
440467
stringBuilder.append(this.isHandleRenamedFiles());
441468
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1123,13 +1123,12 @@ public Map<Repository, Optional<Exception>> prepareIndexer(RuntimeEnvironment en
11231123
* @param repositories list of repository paths relative to source root
11241124
* @return map of repository to exception
11251125
* @throws IndexerException indexer exception
1126-
* @throws IOException I/O exception
11271126
*/
11281127
public Map<Repository, Optional<Exception>> prepareIndexer(RuntimeEnvironment env,
11291128
Set<String> searchPaths,
11301129
boolean addProjects,
11311130
boolean createHistoryCache,
1132-
List<String> repositories) throws IndexerException, IOException {
1131+
List<String> repositories) throws IndexerException {
11331132

11341133
if (!env.validateUniversalCtags()) {
11351134
throw new IndexerException("Could not find working Universal ctags. " +

0 commit comments

Comments
 (0)