IncrementalRegistry.java
package network.ike.docs.plugin.registry;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
/**
* The blob-keyed incremental registry ({@code dev-adoc-packages},
* IKE-Network/ike-issues#1341). Every entry records the blob id of the
* content it was parsed from, and a run decides each file by one rule
* against the registry it wrote last time:
*
* <ul>
* <li>same path and same blob: keep the entry, without opening the file;</li>
* <li>same blob at a path that is gone, a rename: re-key the entry to the
* new path, without opening the file;</li>
* <li>a new or changed blob: parse the file;</li>
* <li>a path that is gone: drop the entry.</li>
* </ul>
*
* <p>Files that are not topics get an entry too, with their blob, so they
* are not opened again either. What is derived across files is derived
* again on every run from the entries: the duplicate-id findings, the
* counts, the domain grouping and the order. The result has the shape of
* {@link TopicRegistry#model}, so it reads back through
* {@link TopicRegistry#load} and serves as the previous registry of the
* next run. A first run, or a run after the previous registry was
* deleted, parses everything.
*
* @since 136
*/
public final class IncrementalRegistry {
/** The {@code registry-version} of a registry whose entries carry blobs. */
public static final String VERSION = "1.3";
/**
* What one update did.
*
* @param model the registry model, in the shape of {@link TopicRegistry#model}
* @param kept entries kept as they were: same path, same blob
* @param rekeyed entries re-keyed to a new path: same blob, path gone
* @param parsed files parsed: a new or changed blob
* @param dropped entries dropped: path gone and blob not seen elsewhere
*/
public record Outcome(Map<String, Object> model, int kept, int rekeyed, int parsed, int dropped) {
}
/** One entry of the previous registry with where it was filed. */
private record Previous(Map<String, Object> entry, boolean topic, boolean assembly, List<String> findings) {
}
/** One entry of the new registry, in path order. */
private record Current(String file, Map<String, Object> entry, boolean topic, boolean assembly,
List<String> findings) {
}
private IncrementalRegistry() {
}
/**
* Update a registry against the files a blob index lists.
*
* @param previous the registry written last time, from
* {@link TopicRegistry#load}, or {@code null} for a first run
* @param root the registry root the index paths are relative to
* @param index every {@code .adoc} file under the root with its blob id
* @param base the directory the root is shown relative to (the module
* base directory)
* @param now the generation instant
* @return the outcome, with the new model
* @throws IOException if a file that must be parsed cannot be read
*/
public static Outcome update(Map<String, Object> previous, Path root, BlobIndex index, Path base,
Instant now) throws IOException {
Map<String, Previous> byPath = new HashMap<>();
Map<String, List<String>> pathsByBlob = new HashMap<>();
if (previous != null) {
collectPrevious(previous, byPath, pathsByBlob);
}
Set<String> consumed = new HashSet<>();
List<String> toParse = new ArrayList<>();
Map<String, Current> current = new TreeMap<>();
int kept = 0;
int rekeyed = 0;
for (Map.Entry<String, String> file : index.blobs().entrySet()) {
String path = file.getKey();
String blob = file.getValue();
Previous same = byPath.get(path);
if (same != null && blob.equals(same.entry().get("blob"))) {
consumed.add(path);
current.put(path, new Current(path, copy(same.entry()), same.topic(), same.assembly(),
same.findings()));
kept++;
continue;
}
String movedFrom = null;
for (String candidate : pathsByBlob.getOrDefault(blob, List.of())) {
if (!index.blobs().containsKey(candidate) && !consumed.contains(candidate)) {
movedFrom = candidate;
break;
}
}
if (movedFrom != null) {
consumed.add(movedFrom);
Previous moved = byPath.get(movedFrom);
Map<String, Object> entry = copy(moved.entry());
entry.put("file", path);
current.put(path, new Current(path, entry, moved.topic(), moved.assembly(), moved.findings()));
rekeyed++;
continue;
}
toParse.add(path);
}
int dropped = 0;
for (String path : byPath.keySet()) {
if (!index.blobs().containsKey(path) && !consumed.contains(path)) {
dropped++;
}
}
Path real = root.toRealPath();
List<Current> parsed;
try {
parsed = toParse.parallelStream().map(path -> {
try {
return parse(real, path, index.blobs().get(path));
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}).toList();
} catch (UncheckedIOException e) {
throw e.getCause();
}
for (Current c : parsed) {
current.put(c.file(), c);
}
return new Outcome(model(current.values(), index.findings(), real, base, now), kept, rekeyed,
toParse.size(), dropped);
}
/**
* Parse one file into its entry: the header through {@link TopicHeader},
* the index terms through {@link TermIndex}, the blob after the path.
*/
private static Current parse(Path real, String path, String blob) throws IOException {
String content = Files.readString(real.resolve(path), StandardCharsets.UTF_8);
TopicHeader header = TopicHeader.parse(path, content);
Map<String, Object> entry;
if (header.topic()) {
entry = withBlob(TopicRegistry.topicEntry(header, path), blob);
List<String> terms = TermIndex.termsOf(content);
if (!terms.isEmpty()) {
entry.put("index-terms", terms);
}
} else {
entry = withBlob(TopicRegistry.otherEntry(header, path), blob);
}
return new Current(path, entry, header.topic(), header.includes() > 0, header.findings());
}
/** The entry with {@code blob} right after {@code file}, and {@code topic: false} on a file that is not one. */
private static Map<String, Object> withBlob(Map<String, Object> entry, String blob) {
Map<String, Object> out = new LinkedHashMap<>();
boolean topic = entry.containsKey("id");
for (Map.Entry<String, Object> e : entry.entrySet()) {
out.put(e.getKey(), e.getValue());
if (e.getKey().equals("file")) {
if (!topic) {
out.put("topic", false);
}
out.put("blob", blob);
}
}
return out;
}
@SuppressWarnings("unchecked")
private static void collectPrevious(Map<String, Object> previous, Map<String, Previous> byPath,
Map<String, List<String>> pathsByBlob) {
Map<String, List<String>> findingsByFile = new HashMap<>();
for (Object f : (List<Object>) previous.getOrDefault("findings", List.of())) {
String finding = String.valueOf(f);
int colon = finding.indexOf(": ");
if (colon < 0 || isDerived(finding)) {
continue;
}
findingsByFile.computeIfAbsent(finding.substring(0, colon), k -> new ArrayList<>())
.add(finding.substring(colon + 2));
}
for (Object d : (List<Object>) previous.getOrDefault("domains", List.of())) {
for (Object t : (List<Object>) ((Map<String, Object>) d).getOrDefault("topics", List.of())) {
remember((Map<String, Object>) t, true, false, byPath, pathsByBlob, findingsByFile);
}
}
for (Object a : (List<Object>) previous.getOrDefault("assemblies", List.of())) {
remember((Map<String, Object>) a, false, true, byPath, pathsByBlob, findingsByFile);
}
for (Object o : (List<Object>) previous.getOrDefault("other-files", List.of())) {
remember((Map<String, Object>) o, false, false, byPath, pathsByBlob, findingsByFile);
}
}
private static void remember(Map<String, Object> entry, boolean topic, boolean assembly,
Map<String, Previous> byPath, Map<String, List<String>> pathsByBlob,
Map<String, List<String>> findingsByFile) {
String file = String.valueOf(entry.get("file"));
Object blob = entry.get("blob");
if (blob == null || byPath.containsKey(file)) {
return; // an entry without a blob, from a registry of the full scan, is parsed again
}
byPath.put(file, new Previous(entry, topic, assembly,
List.copyOf(findingsByFile.getOrDefault(file, List.of()))));
pathsByBlob.computeIfAbsent(String.valueOf(blob), k -> new ArrayList<>()).add(file);
}
/** A finding derived across files or from the tree, computed afresh on every run rather than carried. */
private static boolean isDerived(String finding) {
return finding.contains(": duplicate id '") || finding.endsWith(": symbolic link, skipped");
}
private static Map<String, Object> copy(Map<String, Object> entry) {
return new LinkedHashMap<>(entry);
}
/**
* Assemble the model from the entries in path order, as
* {@link TopicRegistry#model} assembles it from headers: topics grouped
* by domain and sorted by id (a duplicate id kept in path order), the
* other files split into assemblies and plain files, the tree findings
* first and then each file's, with the duplicate-id findings derived
* again.
*/
private static Map<String, Object> model(Iterable<Current> entries, List<String> treeFindings, Path real,
Path base, Instant now) {
Map<String, Object> registry = new LinkedHashMap<>();
registry.put("registry-version", VERSION);
registry.put("generated", now.truncatedTo(ChronoUnit.SECONDS).toString());
registry.put("scanned-from", base.toAbsolutePath().normalize().toString());
registry.put("roots", List.of(relativeTo(base, real)));
TreeMap<String, List<Map<String, Object>>> domains = new TreeMap<>();
List<Object> assemblies = new ArrayList<>();
List<Object> otherFiles = new ArrayList<>();
List<String> findings = new ArrayList<>(treeFindings);
findings.sort(null);
Map<String, String> firstFileById = new HashMap<>();
int files = 0;
int topics = 0;
for (Current c : entries) {
files++;
if (c.topic()) {
topics++;
String id = String.valueOf(c.entry().get("id"));
domains.computeIfAbsent(TopicRegistry.domainOf(id), k -> new ArrayList<>()).add(c.entry());
String earlier = firstFileById.putIfAbsent(id, c.file());
if (earlier != null) {
findings.add(c.file() + ": duplicate id '" + id + "', already declared by " + earlier);
}
} else {
(c.assembly() ? assemblies : otherFiles).add(c.entry());
}
for (String f : c.findings()) {
findings.add(c.file() + ": " + f);
}
}
registry.put("topic-count", topics);
registry.put("file-count", files);
List<Object> domainList = new ArrayList<>();
for (Map.Entry<String, List<Map<String, Object>>> e : domains.entrySet()) {
Map<String, Object> domain = new LinkedHashMap<>();
domain.put("id", e.getKey());
e.getValue().sort((x, y) -> String.valueOf(x.get("id")).compareTo(String.valueOf(y.get("id"))));
domain.put("topics", e.getValue());
domainList.add(domain);
}
registry.put("domains", domainList);
registry.put("assemblies", assemblies);
registry.put("other-files", otherFiles);
registry.put("findings", findings);
return registry;
}
private static String relativeTo(Path base, Path path) {
try {
Path b = base.toRealPath();
if (path.startsWith(b)) {
String relative = b.relativize(path).toString().replace('\\', '/');
return relative.isEmpty() ? "." : relative;
}
} catch (IOException e) {
// fall through to the absolute form
}
return path.toString();
}
}