TopicRegistryMojo.java
package network.ike.docs.plugin;
import network.ike.docs.plugin.packages.AdocLayout;
import network.ike.docs.plugin.packages.AdocPackages;
import network.ike.docs.plugin.registry.BlobIndex;
import network.ike.docs.plugin.registry.GitBlobIndex;
import network.ike.docs.plugin.registry.IncrementalRegistry;
import network.ike.docs.plugin.registry.TermIndex;
import network.ike.docs.plugin.registry.TopicHeader;
import network.ike.docs.plugin.registry.TopicRegistry;
import org.apache.maven.api.plugin.MojoException;
import org.apache.maven.api.plugin.annotations.Mojo;
import org.apache.maven.api.plugin.annotations.Parameter;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Generate the topic registry: every AsciiDoc file under the module's source
* root, with its storage location and header metadata (the {@code :topic-*:}
* block of a topic fragment; the title, document attributes and include count
* of any file), written to {@code target/topic-registry.yaml} in the shape
* {@code IKE-TOPIC-REGISTRY.md} gives the registry: topics grouped into
* domains by the prefix of their id. The same structure is printed to the
* build log as a tree. It applies to every module the doc pipeline renders:
* topic libraries, assemblies, doc-only and hybrid projects.
*
* <p>This is the topic registry's build-time revision: the registry keeps its
* purpose and shape and is now generated from the files on every run instead
* of kept by hand at {@code src/docs/asciidoc/topic-registry.yaml}. It is
* never committed and never edits a source file. Header findings (a
* missing required attribute, an anchor that does not match the id, an
* unknown type or status, the same id in two files) are reported in the
* registry and the log and do not fail the build.
*
* <p>A module without {@code src/docs/asciidoc} is skipped with one log
* line, so the goal can run from a reactor root. Usage:
*
* <pre>
* mvn idoc:topic-registry # the module's src/docs/asciidoc
* mvn idoc:topic-registry -Dike.topic-registry.roots=a,b # other roots, comma-separated
* mvn idoc:topic-registry -Dike.topic-registry.output=/tmp/r.yaml # write elsewhere
* mvn idoc:topic-registry -Dike.topic-registry.add=topics/x/new.adoc # add one ingested file, no rescan
* mvn idoc:topic-registry -Dike.topic-registry.rootFile=src/docs/asciidoc/index.adoc # only what index.adoc includes
* </pre>
*
* <p>With {@code -Dike.topic-registry.rootFile} the registry holds that file
* and the {@code .adoc} files it reaches through {@code include::} lines,
* recursively; other files under the root are left out. The file must lie
* under one of the roots, and paths stay relative to that root. Includes that
* cannot be followed (an attribute reference such as {@code {topicsdir}/},
* an absolute path, a URI, a missing file) are findings. It cannot be combined
* with {@code add}.
*
* <p>With {@code -Dike.topic-registry.add} the named files are parsed alone
* and merged into the existing registry: placed in their domain, counted, and
* checked for a duplicate id against the registry. An entry for the same file
* is replaced, so re-ingesting is safe. Deleted files are not noticed that
* way; a run without {@code add} re-derives everything.
*
* <p>The default phase is {@code validate}: the goal reads sources and
* writes only under {@code target/}, so an execution declared without a
* phase (as {@code ike-parent} does for {@code lint-site}) refreshes the
* registry on every build, including a bare {@code mvn validate}.
*
* <p>With the POM property {@code ike.adoc.layout=packages}
* ({@code dev-adoc-packages}, IKE-Network/ike-issues#1341) the registry is
* blob-keyed and incremental: {@link GitBlobIndex} lists every file with
* the git blob id of its content without opening it, and
* {@link IncrementalRegistry} keeps the entry of a file whose path and
* blob are unchanged, re-keys the entry of a blob that moved, parses only
* a new or changed blob and drops a path that is gone. A tree without git
* falls back to hashing every file in Java, the full scan's cost. The
* registry and the term index are then written to
* {@code target/adoc-meta/META-INF/ike/<groupId>/<artifactId>/topic-registry.yaml}
* and {@code term-index.yaml}, which the parent POM ships in the adoc zip,
* with every {@code file:} relative to the AsciiDoc root; {@code add} is a
* special case of the blob rule and needs no separate path. Without the
* property nothing above changes.
*
* <p>Skip with {@code -Dike.skip.topic-registry=true}.
*
* @since 109
*/
@Mojo(name = "topic-registry", defaultPhase = "validate")
public class TopicRegistryMojo implements org.apache.maven.api.plugin.Mojo {
@org.apache.maven.api.di.Inject
private org.apache.maven.api.plugin.Log log;
/**
* Access the Maven logger.
*
* @return the logger
*/
protected org.apache.maven.api.plugin.Log getLog() { return log; }
/**
* Directories to scan, comma-separated. Defaults to the module's
* {@code src/docs/asciidoc}; when that default directory does not exist
* the goal is skipped, while a named directory that does not exist
* fails the build.
*/
@Parameter(property = "ike.topic-registry.roots",
defaultValue = "${project.basedir}/src/docs/asciidoc")
String roots;
/** Where the generated registry is written. */
@Parameter(property = "ike.topic-registry.output",
defaultValue = "${project.build.directory}/topic-registry.yaml")
File output;
/** The most {@code .adoc} files one root may hold before the scan fails. */
@Parameter(property = "ike.topic-registry.maxFiles", defaultValue = "5000")
int maxFiles;
/**
* Files to add to the existing registry without rescanning, comma-separated,
* relative to the module base directory or absolute. Each must be an
* {@code .adoc} file under one of the registry's roots. When no generated
* registry exists yet, the full scan runs instead.
*/
@Parameter(property = "ike.topic-registry.add")
String add;
/**
* An {@code .adoc} file to build the registry from instead of scanning
* every file: the file and what it reaches through {@code include::}
* lines. Relative to the module base directory or absolute; it must lie
* under one of the {@link #roots}. Unset, the whole root is scanned.
*/
@Parameter(property = "ike.topic-registry.rootFile")
String rootFile;
/** Skip the goal. */
@Parameter(property = "ike.skip.topic-registry", defaultValue = "false")
boolean skip;
/**
* The layout the library declares: with {@code packages} the registry is
* blob-keyed, incremental, and written with the term index under
* {@code META-INF/ike/<groupId>/<artifactId>/} in {@link #metaDirectory}.
*/
@Parameter(property = AdocLayout.PROPERTY)
String layout;
/**
* Where the packages layout's metadata tree is written; the parent POM
* zips it with the sources. Used only with {@code ike.adoc.layout=packages}.
*/
@Parameter(property = "ike.adoc.meta-directory", defaultValue = "${project.build.directory}/adoc-meta")
File metaDirectory;
/** The build directory, to tell the default {@link #output} from one the user chose. */
@Parameter(defaultValue = "${project.build.directory}", readonly = true)
File buildDirectory;
/** The project this goal runs in; registry paths are shown relative to its base directory. */
@org.apache.maven.api.di.Inject
private org.apache.maven.api.Project project;
/** Creates this goal instance. */
public TopicRegistryMojo() {}
@Override
public void execute() throws MojoException {
if (skip) {
getLog().debug("idoc:topic-registry — skipped");
return;
}
long started = System.nanoTime();
Path base = project.getBasedir();
boolean fromRootFile = rootFile != null && !rootFile.isBlank();
AdocLayout adocLayout;
try {
adocLayout = AdocLayout.of(layout);
} catch (IllegalArgumentException e) {
throw new MojoException(e.getMessage(), e);
}
if (adocLayout == AdocLayout.PACKAGES) {
if (fromRootFile) {
throw new MojoException("-Dike.topic-registry.rootFile is not supported with "
+ AdocLayout.PROPERTY + "=packages; the registry of a packaged library"
+ " covers every file under its root");
}
executePackages(base, started);
return;
}
if (fromRootFile && add != null && !add.isBlank()) {
throw new MojoException("-Dike.topic-registry.rootFile and -Dike.topic-registry.add"
+ " cannot be combined; run with one or the other");
}
if (add != null && !add.isBlank()) {
if (Files.isRegularFile(output.toPath())) {
addToRegistry(base, output.toPath(), started);
return;
}
getLog().info("idoc:topic-registry — no generated registry at " + output
+ " yet, running the full scan");
}
Path defaultRoot = base.resolve("src/docs/asciidoc");
List<Path> rootPaths = new ArrayList<>();
for (String r : roots.split(",")) {
if (!r.isBlank()) {
rootPaths.add(Path.of(r.strip()));
}
}
if (!fromRootFile && rootPaths.size() == 1 && rootPaths.get(0).toAbsolutePath().normalize()
.equals(defaultRoot.toAbsolutePath().normalize())
&& !Files.isDirectory(defaultRoot)) {
getLog().info("idoc:topic-registry — no src/docs/asciidoc in " + base.getFileName()
+ ", skipping");
return;
}
List<TopicRegistry.Scan> scans = new ArrayList<>();
if (fromRootFile) {
scans.add(scanFromRootFile(base, rootPaths));
} else {
for (Path root : rootPaths) {
if (!Files.isDirectory(root)) {
throw new MojoException("Registry root does not exist: " + root
+ " (check -Dike.topic-registry.roots)");
}
try {
scans.add(TopicRegistry.scan(root, maxFiles));
} catch (IOException e) {
throw new MojoException("Cannot scan " + root + ": " + e.getMessage(), e);
} catch (IllegalStateException e) {
throw new MojoException(e.getMessage(), e);
}
}
}
String yaml = TopicRegistry.yaml(TopicRegistry.model(scans, base, Instant.now()));
Path out = output.toPath();
try {
Files.createDirectories(out.getParent());
Files.writeString(out, yaml, StandardCharsets.UTF_8);
} catch (IOException e) {
throw new MojoException("Cannot write the registry to " + out + ": " + e.getMessage(), e);
}
int topics = 0;
int others = 0;
int findings = 0;
for (TopicRegistry.Scan scan : scans) {
for (String line : TopicRegistry.tree(scan, base)) {
getLog().info(line);
}
topics += scan.topics().size();
others += scan.others().size();
findings += scan.findings().size();
for (String finding : scan.findings()) {
getLog().warn("registry: " + finding);
}
}
long millis = (System.nanoTime() - started) / 1_000_000;
getLog().info(topics + " topic" + (topics == 1 ? "" : "s") + ", " + others
+ " other file" + (others == 1 ? "" : "s") + ", " + findings
+ " finding" + (findings == 1 ? "" : "s") + "; topic registry written to "
+ shown(base, out) + " (" + millis + " ms)");
}
/**
* The packages layout's run: one root, the blob index from git or by
* hashing, the incremental update against the registry written last
* time, the registry and the term index under {@code META-INF}.
*/
private void executePackages(Path base, long started) throws MojoException {
List<Path> rootPaths = new ArrayList<>();
for (String r : roots.split(",")) {
if (!r.isBlank()) {
rootPaths.add(Path.of(r.strip()));
}
}
if (rootPaths.size() != 1) {
throw new MojoException(AdocLayout.PROPERTY + "=packages takes exactly one registry root,"
+ " the library's AsciiDoc root; got " + rootPaths + " (check -Dike.topic-registry.roots)");
}
Path root = rootPaths.get(0);
if (!Files.isDirectory(root)) {
if (root.toAbsolutePath().normalize().equals(base.resolve("src/docs/asciidoc").toAbsolutePath().normalize())) {
getLog().info("idoc:topic-registry — no src/docs/asciidoc in " + base.getFileName() + ", skipping");
return;
}
throw new MojoException("Registry root does not exist: " + root + " (check -Dike.topic-registry.roots)");
}
Path metaDir = metaDirectory.toPath().resolve(AdocPackages.metaDirectory(project.getGroupId(),
project.getArtifactId()));
Path defaultOutput = buildDirectory.toPath().resolve("topic-registry.yaml").toAbsolutePath().normalize();
Path out = output.toPath().toAbsolutePath().normalize().equals(defaultOutput)
? metaDir.resolve("topic-registry.yaml") : output.toPath();
Path termsOut = out.resolveSibling("term-index.yaml");
Map<String, Object> previous = null;
if (Files.isRegularFile(out)) {
try {
previous = TopicRegistry.load(out);
} catch (IOException e) {
getLog().warn("idoc:topic-registry — cannot read the previous registry " + shown(base, out)
+ " (" + e.getMessage() + "); parsing every file");
}
}
if (add != null && !add.isBlank()) {
getLog().info("idoc:topic-registry — -Dike.topic-registry.add is a special case of the blob rule"
+ " under " + AdocLayout.PROPERTY + "=packages: changed files are parsed, the rest kept");
}
BlobIndex index;
IncrementalRegistry.Outcome outcome;
try {
index = GitBlobIndex.read(root).orElse(null);
if (index == null) {
getLog().info("idoc:topic-registry — " + shown(base, root) + " is not in a git working tree;"
+ " hashing every file instead");
index = BlobIndex.hashed(root);
}
if (index.blobs().size() > maxFiles) {
throw new MojoException("More than " + maxFiles + " .adoc files under " + root
+ "; raise -Dike.topic-registry.maxFiles");
}
outcome = IncrementalRegistry.update(previous, root, index, base, Instant.now());
} catch (IOException e) {
throw new MojoException("Cannot index " + root + ": " + e.getMessage(), e);
}
Map<String, Object> model = outcome.model();
try {
Files.createDirectories(out.getParent());
Files.writeString(out, TopicRegistry.yaml(model), StandardCharsets.UTF_8);
Files.createDirectories(termsOut.getParent());
Files.writeString(termsOut, TermIndex.yaml(TermIndex.model(model, Instant.now())), StandardCharsets.UTF_8);
} catch (IOException e) {
throw new MojoException("Cannot write the registry to " + out + ": " + e.getMessage(), e);
}
@SuppressWarnings("unchecked")
List<Object> findings = (List<Object>) model.get("findings");
for (Object f : findings) {
getLog().warn("registry: " + f);
}
int topics = (Integer) model.get("topic-count");
int others = (Integer) model.get("file-count") - topics;
long millis = (System.nanoTime() - started) / 1_000_000;
getLog().info(topics + " topic" + (topics == 1 ? "" : "s") + ", " + others
+ " other file" + (others == 1 ? "" : "s") + ", " + findings.size()
+ " finding" + (findings.size() == 1 ? "" : "s") + "; " + outcome.kept() + " kept, "
+ outcome.rekeyed() + " re-keyed, " + outcome.parsed() + " parsed, " + outcome.dropped()
+ " dropped by " + (index.source() == BlobIndex.Source.GIT ? "git blob" : "hashed blob")
+ "; topic registry and term index written to " + shown(base, out.getParent())
+ "/ (" + millis + " ms)");
}
private TopicRegistry.Scan scanFromRootFile(Path base, List<Path> rootPaths) throws MojoException {
Path file = base.resolve(rootFile.strip());
if (!Files.isRegularFile(file)) {
throw new MojoException("Root file does not exist: " + file
+ " (check -Dike.topic-registry.rootFile)");
}
try {
Path realFile = file.toRealPath();
for (Path root : rootPaths) {
if (Files.isDirectory(root) && realFile.startsWith(root.toRealPath())) {
getLog().info("idoc:topic-registry — from root file " + shown(base, file));
return TopicRegistry.scanFrom(root, file);
}
}
} catch (IOException e) {
throw new MojoException("Cannot scan from " + file + ": " + e.getMessage(), e);
} catch (IllegalArgumentException e) {
throw new MojoException(e.getMessage(), e);
}
throw new MojoException("Root file " + file + " is under none of the registry roots "
+ rootPaths + " (check -Dike.topic-registry.roots)");
}
private void addToRegistry(Path base, Path out, long started) throws MojoException {
List<Path> files = new ArrayList<>();
for (String f : add.split(",")) {
if (!f.isBlank()) {
files.add(Path.of(f.strip()));
}
}
Map<String, Object> model;
try {
model = TopicRegistry.load(out);
} catch (IOException e) {
throw new MojoException("Cannot read the generated registry " + out + ": "
+ e.getMessage(), e);
}
List<TopicHeader> added;
try {
added = TopicRegistry.add(model, base, files, Instant.now());
} catch (IllegalArgumentException e) {
throw new MojoException(e.getMessage(), e);
} catch (IOException e) {
throw new MojoException("Cannot read a file to add: " + e.getMessage(), e);
}
try {
Files.writeString(out, TopicRegistry.yaml(model), StandardCharsets.UTF_8);
} catch (IOException e) {
throw new MojoException("Cannot write the registry to " + out + ": " + e.getMessage(), e);
}
for (TopicHeader h : added) {
getLog().info("registry: " + (h.topic() ? "topic " + h.id() + " at " : "file ") + h.file());
}
@SuppressWarnings("unchecked")
List<Object> findings = (List<Object>) model.get("findings");
for (Object f : findings) {
for (TopicHeader h : added) {
if (String.valueOf(f).startsWith(h.file() + ": ")) {
getLog().warn("registry: " + f);
}
}
}
long millis = (System.nanoTime() - started) / 1_000_000;
getLog().info("added " + added.size() + " file" + (added.size() == 1 ? "" : "s")
+ " to " + shown(base, out) + " without rescanning (" + millis + " ms)");
}
private static String shown(Path base, Path out) {
return out.toAbsolutePath().startsWith(base.toAbsolutePath())
? base.toAbsolutePath().relativize(out.toAbsolutePath()).toString()
: out.toString();
}
}