Source code

001/*
002 * Extra Bnd Repository Plugins
003 * Copyright (C) 2019-2021 Michael N. Lipp
004 * 
005 * This program is free software; you can redistribute it and/or modify it 
006 * under the terms of the GNU Affero General Public License as published by 
007 * the Free Software Foundation; either version 3 of the License, or 
008 * (at your option) any later version.
009 * 
010 * This program is distributed in the hope that it will be useful, but 
011 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
012 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License 
013 * for more details.
014 * 
015 * You should have received a copy of the GNU Affero General Public License along 
016 * with this program; if not, see <http://www.gnu.org/licenses/>.
017 */
018
019package de.mnl.osgi.bnd.repository.maven.idxmvn;
020
021import aQute.bnd.http.HttpClient;
022import aQute.bnd.osgi.Processor;
023import aQute.bnd.osgi.repository.ResourcesRepository;
024import aQute.maven.api.Archive;
025import aQute.maven.provider.MavenBackingRepository;
026import aQute.service.reporter.Reporter;
027import de.mnl.osgi.bnd.maven.MavenResourceRepository;
028import static de.mnl.osgi.bnd.maven.RepositoryUtils.rethrow;
029import static de.mnl.osgi.bnd.maven.RepositoryUtils.unthrow;
030import java.io.File;
031import java.io.IOException;
032import java.io.OutputStream;
033import java.net.URL;
034import java.nio.file.Files;
035import java.nio.file.Path;
036import java.util.ArrayList;
037import java.util.Arrays;
038import java.util.Collections;
039import java.util.Comparator;
040import java.util.HashMap;
041import java.util.Iterator;
042import java.util.List;
043import java.util.Map;
044import java.util.Optional;
045import java.util.concurrent.CompletableFuture;
046import java.util.concurrent.CompletionException;
047import java.util.concurrent.ConcurrentHashMap;
048import java.util.concurrent.ExecutorService;
049import java.util.concurrent.Executors;
050import java.util.concurrent.atomic.AtomicBoolean;
051import javax.xml.parsers.DocumentBuilderFactory;
052import javax.xml.parsers.ParserConfigurationException;
053import javax.xml.transform.OutputKeys;
054import javax.xml.transform.Transformer;
055import javax.xml.transform.TransformerException;
056import javax.xml.transform.TransformerFactory;
057import javax.xml.transform.dom.DOMSource;
058import javax.xml.transform.stream.StreamResult;
059import org.osgi.resource.Resource;
060import org.osgi.service.repository.Repository;
061import org.slf4j.Logger;
062import org.slf4j.LoggerFactory;
063import org.w3c.dom.Document;
064import org.w3c.dom.Element;
065
066/**
067 * Provide an OSGi repository (a collection of {@link Resource}s, see 
068 * {@link Repository}), filled with resolved artifacts from given groupIds.
069 */
070@SuppressWarnings({ "PMD.DataflowAnomalyAnalysis", "PMD.TooManyFields",
071    "PMD.FieldNamingConventions" })
072public class IndexedMavenRepository extends ResourcesRepository {
073
074    /* default */ static final ExecutorService programLoaders
075        = Executors.newFixedThreadPool(4);
076    /* default */ static final ExecutorService revisionLoaders
077        = Executors.newFixedThreadPool(8);
078
079    private static final Logger LOG = LoggerFactory.getLogger(
080        IndexedMavenRepository.class);
081    private final String name;
082    private final Path indexDbDir;
083    private final Path depsDir;
084    private final List<URL> releaseUrls;
085    private final List<URL> snapshotUrls;
086    private final File localRepo;
087    private final Reporter reporter;
088    private final HttpClient client;
089    private final boolean logIndexing;
090    private final ExecutorService groupLoaders
091        = Executors.newFixedThreadPool(4);
092    private final MavenResourceRepository mavenRepository;
093    private final Map<String, MavenGroupRepository> groups
094        = new ConcurrentHashMap<>();
095    private Map<String, MavenGroupRepository> backupGroups;
096    private final AtomicBoolean refreshing = new AtomicBoolean();
097
098    /**
099     * Create a new instance that uses the provided information/resources to perform
100     * its work.
101     *
102     * @param name the name
103     * @param releaseUrls the release urls
104     * @param snapshotUrls the snapshot urls
105     * @param localRepo the local Maven repository (cache)
106     * @param indexDbDir the persistent representation of this repository's content
107     * @param reporter a reporter for reporting the progress
108     * @param client an HTTP client for obtaining information from the Nexus server
109     * @param logIndexing the log indexing
110     * @throws Exception the exception
111     */
112    @SuppressWarnings({ "PMD.AvoidCatchingGenericException",
113        "PMD.SignatureDeclareThrowsException", "PMD.AvoidDuplicateLiterals" })
114    public IndexedMavenRepository(String name, List<URL> releaseUrls,
115            List<URL> snapshotUrls, File localRepo, File indexDbDir,
116            Reporter reporter, HttpClient client, boolean logIndexing)
117            throws Exception {
118        this.name = name;
119        this.indexDbDir = indexDbDir.toPath();
120        depsDir = this.indexDbDir.resolve("dependencies");
121        this.releaseUrls = releaseUrls;
122        this.snapshotUrls = snapshotUrls;
123        this.localRepo = localRepo;
124        this.reporter = reporter;
125        this.client = client;
126        this.logIndexing = logIndexing;
127
128        // Check prerequisites
129        if (indexDbDir.exists() && !indexDbDir.isDirectory()) {
130            reporter.error("%s must be a directory.", indexDbDir);
131            throw new IOException(indexDbDir + "must be a directory.");
132        }
133        if (!indexDbDir.exists()) {
134            indexDbDir.mkdirs();
135        }
136        if (!depsDir.toFile().exists()) {
137            depsDir.toFile().mkdir();
138        }
139
140        // Our repository
141        mavenRepository = createMavenRepository();
142
143        // Restore
144        @SuppressWarnings("PMD.UseConcurrentHashMap")
145        Map<String, MavenGroupRepository> oldGroups = new HashMap<>();
146        CompletableFuture
147            .allOf(scanRequested(oldGroups), scanDependencies(oldGroups)).get();
148        for (MavenGroupRepository groupRepo : groups.values()) {
149            addAll(groupRepo.getResources());
150        }
151        backupGroups = groups;
152    }
153
154    @SuppressWarnings({ "PMD.SignatureDeclareThrowsException", "resource" })
155    private MavenResourceRepository createMavenRepository() throws Exception {
156        // Create repository from URLs
157        List<MavenBackingRepository> releaseBackers = new ArrayList<>();
158        for (URL url : releaseUrls) {
159            releaseBackers.addAll(MavenBackingRepository.create(
160                url.toString(), reporter, localRepo, client));
161        }
162        List<MavenBackingRepository> snapshotBackers = new ArrayList<>();
163        for (URL url : snapshotUrls) {
164            snapshotBackers.addAll(MavenBackingRepository.create(
165                url.toString(), reporter, localRepo, client));
166        }
167        return new MavenResourceRepository(
168            localRepo, name(), releaseBackers,
169            snapshotBackers, Processor.getExecutor(), reporter)
170                .setResourceSupplier(this::restoreResource);
171    }
172
173    private Optional<Resource> restoreResource(Archive archive) {
174        return Optional.ofNullable(backupGroups.get(archive.revision.group))
175            .flatMap(group -> group.searchInBackup(archive));
176    }
177
178    /**
179     * Return the name of this repository.
180     * 
181     * @return the name;
182     */
183    public final String name() {
184        return name;
185    }
186
187    /**
188     * Return the representation of this repository in the local file system.
189     * 
190     * @return the location
191     */
192    public final File location() {
193        return indexDbDir.toFile();
194    }
195
196    /**
197     * Returns true if indexing is to be logged.
198     *
199     * @return true, if indexing should be log
200     */
201    public boolean logIndexing() {
202        return logIndexing;
203    }
204
205    /**
206     * Return the Maven repository object used to implements this repository.
207     *
208     * @return the repository 
209     */
210    public MavenResourceRepository mavenRepository() {
211        return mavenRepository;
212    }
213
214    /**
215     * Get or create the group repository for the given group id.
216     * If the repository is created, it is created as a repository
217     * for dependencies.
218     *
219     * @param groupId the group id
220     * @return the repository
221     * @throws IOException Signals that an I/O exception has occurred.
222     */
223    public MavenGroupRepository getOrCreateGroupRepository(String groupId)
224            throws IOException {
225        @SuppressWarnings("PMD.PrematureDeclaration")
226        MavenGroupRepository result = rethrow(IOException.class,
227            () -> groups.computeIfAbsent(groupId,
228                grp -> unthrow(() -> new MavenGroupRepository(grp,
229                    depsDir.resolve(grp), false, this, client, reporter))));
230        return result;
231    }
232
233    /**
234     * Refresh this repository's content.
235     * 
236     * @return true if refreshed, false if not refreshed possibly due to error
237     * @throws Exception if a problem occurs
238     */
239    @SuppressWarnings("PMD.SignatureDeclareThrowsException")
240    public boolean refresh() throws Exception {
241        if (!refreshing.compareAndSet(false, true)) {
242            reporter.warning("Repository is already refreshing.");
243            return false;
244        }
245        String threadName = Thread.currentThread().getName();
246        try {
247            Thread.currentThread().setName("IndexedMaven Refresher");
248            return doRefresh();
249        } finally {
250            Thread.currentThread().setName(threadName);
251            refreshing.set(false);
252        }
253    }
254
255    @SuppressWarnings({ "PMD.AvoidInstantiatingObjectsInLoops",
256        "PMD.AvoidDuplicateLiterals", "PMD.SignatureDeclareThrowsException",
257        "PMD.CognitiveComplexity", "PMD.NPathComplexity" })
258    private boolean doRefresh() throws Exception {
259        mavenRepository.reset();
260
261        // Reuse and clear (or create new) group repositories for the existing
262        // directories, first for explicitly requested group ids...
263        backupGroups = new HashMap<>(groups);
264        groups.clear();
265        for (var group : backupGroups.values()) {
266            group.reset();
267        }
268
269        // Scan requested first to avoid problems if a group is moved from
270        // dependency to requested.
271        scanRequested(backupGroups).get();
272        scanDependencies(backupGroups).get();
273
274        // Refresh them all.
275        @SuppressWarnings("PMD.GuardLogStatement")
276        CompletableFuture<?>[] repoLoaders
277            = new ArrayList<>(groups.values()).stream()
278                .map(repo -> CompletableFuture.runAsync(() -> {
279                    String threadName = Thread.currentThread().getName();
280                    try {
281                        Thread.currentThread()
282                            .setName("RepoLoader " + repo.id());
283                        LOG.debug("Reloading %s...", repo.id());
284                        repo.reload();
285                    } catch (IOException e) {
286                        throw new CompletionException(e);
287                    } finally {
288                        Thread.currentThread().setName(threadName);
289                    }
290                }, groupLoaders))
291                .toArray(CompletableFuture[]::new);
292        CompletableFuture.allOf(repoLoaders).get();
293        // Remove no longer required group repositories.
294        for (Iterator<Map.Entry<String, MavenGroupRepository>> iter
295            = groups.entrySet().iterator(); iter.hasNext();) {
296            Map.Entry<String, MavenGroupRepository> entry = iter.next();
297            if (entry.getValue().removeIfRedundant()) {
298                iter.remove();
299            }
300        }
301        CompletableFuture.allOf(
302            // Persist updated data.
303            CompletableFuture.allOf(new ArrayList<>(groups.values()).stream()
304                .map(repo -> CompletableFuture.runAsync(() -> {
305                    try {
306                        repo.flush();
307                    } catch (IOException e) {
308                        throw new CompletionException(e);
309                    }
310                }, groupLoaders))
311                .toArray(CompletableFuture[]::new)),
312            // Write federated index.
313            CompletableFuture.runAsync(() -> {
314                if (groups.keySet().equals(backupGroups.keySet())) {
315                    return;
316                }
317                try (OutputStream fos
318                    = Files.newOutputStream(indexDbDir.resolve("index.xml"))) {
319                    writeFederatedIndex(fos);
320                } catch (IOException e) {
321                    throw new CompletionException(e);
322                }
323            }, groupLoaders),
324            // Add collected to root (this).
325            CompletableFuture.runAsync(() -> {
326                set(Collections.emptyList());
327                for (MavenGroupRepository groupRepo : groups.values()) {
328                    addAll(groupRepo.getResources());
329                }
330            }, groupLoaders)).get();
331        backupGroups = groups;
332        return true;
333    }
334
335    private CompletableFuture<Void>
336            scanRequested(Map<String, MavenGroupRepository> oldGroups) {
337        return CompletableFuture.allOf(
338            Arrays.stream(indexDbDir.toFile().list()).parallel()
339                .filter(dir -> dir.matches("^[A-Za-z].*")
340                    && !"index.xml".equals(dir)
341                    && !"dependencies".equals(dir))
342                .map(groupId -> CompletableFuture
343                    .runAsync(() -> restoreGroup(oldGroups, groupId, true),
344                        groupLoaders))
345                .toArray(CompletableFuture[]::new));
346    }
347
348    private CompletableFuture<Void>
349            scanDependencies(Map<String, MavenGroupRepository> oldGroups) {
350        if (!depsDir.toFile().canRead() || !depsDir.toFile().isDirectory()) {
351            return CompletableFuture.completedFuture(null);
352        }
353        return CompletableFuture.allOf(
354            Arrays.stream(depsDir.toFile().list()).parallel()
355                .filter(dir -> dir.matches("^[A-Za-z].*"))
356                .filter(dir -> {
357                    if (groups.containsKey(dir)) {
358                        // Is/has become explicitly requested
359                        try {
360                            Files.walk(depsDir.resolve(dir))
361                                .sorted(Comparator.reverseOrder())
362                                .map(Path::toFile)
363                                .forEach(File::delete);
364                        } catch (IOException e) {
365                            throw new IllegalStateException(e);
366                        }
367                        return false;
368                    }
369                    return true;
370                })
371                .map(groupId -> CompletableFuture
372                    .runAsync(() -> restoreGroup(oldGroups, groupId, false),
373                        groupLoaders))
374                .toArray(CompletableFuture[]::new));
375    }
376
377    private void restoreGroup(Map<String, MavenGroupRepository> oldGroups,
378            String groupId, boolean requested) {
379        if (oldGroups.containsKey(groupId)) {
380            // Reuse existing.
381            MavenGroupRepository groupRepo = oldGroups.get(groupId);
382            groupRepo.reset((requested ? indexDbDir : depsDir).resolve(groupId),
383                requested);
384            groups.put(groupId, groupRepo);
385            return;
386        }
387        try {
388            MavenGroupRepository groupRepo
389                = new MavenGroupRepository(groupId,
390                    (requested ? indexDbDir : depsDir).resolve(groupId),
391                    requested, this, client, reporter);
392            groups.put(groupId, groupRepo);
393        } catch (IOException e) {
394            reporter.exception(e,
395                "Cannot create group repository for %s: %s",
396                groupId, e.getMessage());
397        }
398    }
399
400    private void writeFederatedIndex(OutputStream out) {
401        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
402        dbf.setNamespaceAware(true);
403        Document doc;
404        try {
405            doc = dbf.newDocumentBuilder().newDocument();
406        } catch (ParserConfigurationException e) {
407            return;
408        }
409        @SuppressWarnings("PMD.AvoidFinalLocalVariable")
410        final String repoNs = "http://www.osgi.org/xmlns/repository/v1.0.0";
411        // <repository name=... increment=...>
412        Element repoNode = doc.createElementNS(repoNs, "repository");
413        repoNode.setAttribute("name", name);
414        repoNode.setAttribute("increment",
415            Long.toString(System.currentTimeMillis()));
416        doc.appendChild(repoNode);
417        for (Map.Entry<String, MavenGroupRepository> repo : groups.entrySet()) {
418            // <referral url=...>
419            Element referral = doc.createElementNS(repoNs, "referral");
420            referral.setAttribute("url",
421                (repo.getValue().isRequested() ? "" : "dependencies/")
422                    + repo.getKey() + "/index.xml");
423            repoNode.appendChild(referral);
424        }
425        // Write federated index.
426        try {
427            Transformer transformer
428                = TransformerFactory.newInstance().newTransformer();
429            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
430            DOMSource source = new DOMSource(doc);
431            transformer.transform(source, new StreamResult(out));
432        } catch (TransformerException e) {
433            reporter.exception(e, "Cannot write federated index: %s",
434                e.getMessage());
435        }
436    }
437
438}