001/*
002 * Extra Bnd Repository Plugins
003 * Copyright (C) 2019  Michael N. Lipp
004 * 
005 * This program is free software; you can redistribute it and/or modify it 
006 * under the terms of the GNU Affero General Public License as published by 
007 * the Free Software Foundation; either version 3 of the License, or 
008 * (at your option) any later version.
009 * 
010 * This program is distributed in the hope that it will be useful, but 
011 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
012 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License 
013 * for more details.
014 * 
015 * You should have received a copy of the GNU Affero General Public License along 
016 * with this program; if not, see <http://www.gnu.org/licenses/>.
017 */
018
019package de.mnl.osgi.bnd.repository.maven.idxmvn;
020
021import aQute.bnd.http.HttpClient;
022import aQute.bnd.osgi.Processor;
023import aQute.bnd.osgi.repository.ResourcesRepository;
024import aQute.maven.api.Revision;
025import aQute.maven.provider.MavenBackingRepository;
026import aQute.service.reporter.Reporter;
027import de.mnl.osgi.bnd.maven.MavenResourceRepository;
028
029import static de.mnl.osgi.bnd.maven.RepositoryUtils.rethrow;
030import static de.mnl.osgi.bnd.maven.RepositoryUtils.unthrow;
031
032import java.io.File;
033import java.io.IOException;
034import java.io.OutputStream;
035import java.net.URL;
036import java.nio.file.Files;
037import java.nio.file.Path;
038import java.util.ArrayList;
039import java.util.Arrays;
040import java.util.Collections;
041import java.util.HashMap;
042import java.util.Iterator;
043import java.util.List;
044import java.util.Map;
045import java.util.Optional;
046import java.util.concurrent.CompletableFuture;
047import java.util.concurrent.CompletionException;
048import java.util.concurrent.ConcurrentHashMap;
049import java.util.concurrent.ExecutorService;
050import java.util.concurrent.Executors;
051import java.util.concurrent.atomic.AtomicBoolean;
052
053import javax.xml.parsers.DocumentBuilderFactory;
054import javax.xml.parsers.ParserConfigurationException;
055import javax.xml.transform.OutputKeys;
056import javax.xml.transform.Transformer;
057import javax.xml.transform.TransformerException;
058import javax.xml.transform.TransformerFactory;
059import javax.xml.transform.dom.DOMSource;
060import javax.xml.transform.stream.StreamResult;
061
062import org.osgi.resource.Resource;
063import org.osgi.service.repository.Repository;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066import org.w3c.dom.Document;
067import org.w3c.dom.Element;
068
069/**
070 * Provide an OSGi repository (a collection of {@link Resource}s, see 
071 * {@link Repository}), filled with resolved artifacts from given groupIds.
072 */
073@SuppressWarnings("PMD.DataflowAnomalyAnalysis")
074public class IndexedMavenRepository extends ResourcesRepository {
075
076    /* default */ static ExecutorService programLoaders
077        = Executors.newFixedThreadPool(4);
078    /* default */ static ExecutorService revisionLoaders
079        = Executors.newFixedThreadPool(8);
080
081    private static final Logger LOG = LoggerFactory.getLogger(
082        IndexedMavenRepository.class);
083    private final String name;
084    private final Path indexDbDir;
085    private final Path depsDir;
086    private final List<URL> releaseUrls;
087    private final List<URL> snapshotUrls;
088    private final File localRepo;
089    private final Reporter reporter;
090    private final HttpClient client;
091    private final boolean logIndexing;
092    private final ExecutorService groupLoaders
093        = Executors.newFixedThreadPool(4);
094    private final MavenResourceRepository mavenRepository;
095    private final Map<String, MavenGroupRepository> groups
096        = new ConcurrentHashMap<>();
097    private Map<String, MavenGroupRepository> backupGroups;
098    private final AtomicBoolean refreshing = new AtomicBoolean();
099
100    /**
101     * Create a new instance that uses the provided information/resources to perform
102     * its work.
103     *
104     * @param name the name
105     * @param releaseUrls the release urls
106     * @param snapshotUrls the snapshot urls
107     * @param localRepo the local Maven repository (cache)
108     * @param indexDbDir the persistent representation of this repository's content
109     * @param reporter a reporter for reporting the progress
110     * @param client an HTTP client for obtaining information from the Nexus server
111     * @param logIndexing the log indexing
112     * @throws Exception the exception
113     */
114    @SuppressWarnings({ "PMD.AvoidCatchingGenericException",
115        "PMD.SignatureDeclareThrowsException", "PMD.AvoidDuplicateLiterals" })
116    public IndexedMavenRepository(String name, List<URL> releaseUrls,
117            List<URL> snapshotUrls, File localRepo, File indexDbDir,
118            Reporter reporter, HttpClient client, boolean logIndexing)
119            throws Exception {
120        this.name = name;
121        this.indexDbDir = indexDbDir.toPath();
122        depsDir = this.indexDbDir.resolve("dependencies");
123        this.releaseUrls = releaseUrls;
124        this.snapshotUrls = snapshotUrls;
125        this.localRepo = localRepo;
126        this.reporter = reporter;
127        this.client = client;
128        this.logIndexing = logIndexing;
129
130        // Check prerequisites
131        if (indexDbDir.exists() && !indexDbDir.isDirectory()) {
132            reporter.error("%s must be a directory.", indexDbDir);
133            throw new IOException(indexDbDir + "must be a directory.");
134        }
135        if (!indexDbDir.exists()) {
136            indexDbDir.mkdirs();
137        }
138        if (!depsDir.toFile().exists()) {
139            depsDir.toFile().mkdir();
140        }
141
142        // Our repository
143        mavenRepository = createMavenRepository();
144
145        // Restore
146        @SuppressWarnings("PMD.UseConcurrentHashMap")
147        Map<String, MavenGroupRepository> oldGroups = new HashMap<>();
148        CompletableFuture
149            .allOf(scanRequested(oldGroups), scanDependencies(oldGroups)).get();
150        for (MavenGroupRepository groupRepo : groups.values()) {
151            addAll(groupRepo.getResources());
152        }
153        backupGroups = groups;
154    }
155
156    @SuppressWarnings({ "PMD.SignatureDeclareThrowsException", "resource" })
157    private MavenResourceRepository createMavenRepository() throws Exception {
158        // Create repository from URLs
159        List<MavenBackingRepository> releaseBackers = new ArrayList<>();
160        for (URL url : releaseUrls) {
161            releaseBackers.addAll(MavenBackingRepository.create(
162                url.toString(), reporter, localRepo, client));
163        }
164        List<MavenBackingRepository> snapshotBackers = new ArrayList<>();
165        for (URL url : snapshotUrls) {
166            snapshotBackers.addAll(MavenBackingRepository.create(
167                url.toString(), reporter, localRepo, client));
168        }
169        return new MavenResourceRepository(
170            localRepo, name(), releaseBackers,
171            snapshotBackers, Processor.getExecutor(), reporter)
172                .setResourceSupplier(this::restoreResource);
173    }
174
175    private Optional<Resource> restoreResource(Revision revision) {
176        return Optional.ofNullable(backupGroups.get(revision.group))
177            .flatMap(group -> group.searchInBackup(revision));
178    }
179
180    /**
181     * Return the name of this repository.
182     * 
183     * @return the name;
184     */
185    public final String name() {
186        return name;
187    }
188
189    /**
190     * Return the representation of this repository in the local file system.
191     * 
192     * @return the location
193     */
194    public final File location() {
195        return indexDbDir.toFile();
196    }
197
198    /**
199     * Returns true if indexing is to be logged.
200     *
201     * @return true, if indexing should be log
202     */
203    public boolean logIndexing() {
204        return logIndexing;
205    }
206
207    /**
208     * Return the Maven repository object used to implements this repository.
209     *
210     * @return the repository 
211     */
212    public MavenResourceRepository mavenRepository() {
213        return mavenRepository;
214    }
215
216    /**
217     * Get or create the group repository for the given group id.
218     * If the repository is created, it is created as a repository
219     * for dependencies.
220     *
221     * @param groupId the group id
222     * @return the repository
223     * @throws IOException Signals that an I/O exception has occurred.
224     */
225    public MavenGroupRepository getOrCreateGroupRepository(String groupId)
226            throws IOException {
227        @SuppressWarnings("PMD.PrematureDeclaration")
228        MavenGroupRepository result = rethrow(IOException.class,
229            () -> groups.computeIfAbsent(groupId,
230                grp -> unthrow(() -> new MavenGroupRepository(grp,
231                    depsDir.resolve(grp), false, this, client, reporter))));
232        return result;
233    }
234
235    /**
236     * Refresh this repository's content.
237     * 
238     * @return true if refreshed, false if not refreshed possibly due to error
239     * @throws Exception if a problem occurs
240     */
241    @SuppressWarnings("PMD.SignatureDeclareThrowsException")
242    public boolean refresh() throws Exception {
243        if (!refreshing.compareAndSet(false, true)) {
244            reporter.warning("Repository is already refreshing.");
245            return false;
246        }
247        try {
248            return doRefresh();
249        } finally {
250            refreshing.set(false);
251        }
252    }
253
254    @SuppressWarnings({ "PMD.AvoidInstantiatingObjectsInLoops",
255        "PMD.AvoidDuplicateLiterals", "PMD.SignatureDeclareThrowsException" })
256    private boolean doRefresh() throws Exception {
257        mavenRepository.reset();
258        backupGroups = new HashMap<>(groups);
259
260        // Reuse and clear (or create new) group repositories for the existing
261        // directories, first for explicitly requested group ids...
262        groups.clear();
263        CompletableFuture
264            .allOf(scanRequested(backupGroups), scanDependencies(backupGroups))
265            .get();
266        // Refresh them all.
267        CompletableFuture.allOf(new ArrayList<>(groups.values()).stream()
268            .map(repo -> CompletableFuture.runAsync(() -> {
269                try {
270                    LOG.debug("Reloading %s...", repo.id());
271                    repo.reload();
272                } catch (IOException e) {
273                    throw new CompletionException(e);
274                }
275            }, groupLoaders))
276            .toArray(CompletableFuture[]::new)).get();
277        // Remove no longer required group repositories.
278        for (Iterator<Map.Entry<String, MavenGroupRepository>> iter
279            = groups.entrySet().iterator(); iter.hasNext();) {
280            Map.Entry<String, MavenGroupRepository> entry = iter.next();
281            if (entry.getValue().removeIfRedundant()) {
282                iter.remove();
283            }
284        }
285        CompletableFuture.allOf(
286            // Persist updated data.
287            CompletableFuture.allOf(new ArrayList<>(groups.values()).stream()
288                .map(repo -> CompletableFuture.runAsync(() -> {
289                    try {
290                        repo.flush();
291                    } catch (IOException e) {
292                        throw new CompletionException(e);
293                    }
294                }, groupLoaders))
295                .toArray(CompletableFuture[]::new)),
296            // Write federated index.
297            CompletableFuture.runAsync(() -> {
298                if (groups.keySet().equals(backupGroups.keySet())) {
299                    return;
300                }
301                try (OutputStream fos
302                    = Files.newOutputStream(indexDbDir.resolve("index.xml"))) {
303                    writeFederatedIndex(fos);
304                } catch (IOException e) {
305                    throw new CompletionException(e);
306                }
307            }, groupLoaders),
308            // Add collected to root (this).
309            CompletableFuture.runAsync(() -> {
310                set(Collections.emptyList());
311                for (MavenGroupRepository groupRepo : groups.values()) {
312                    addAll(groupRepo.getResources());
313                }
314            }, groupLoaders)).get();
315        backupGroups = groups;
316        return true;
317    }
318
319    private CompletableFuture<Void>
320            scanRequested(Map<String, MavenGroupRepository> oldGroups) {
321        return CompletableFuture.allOf(
322            Arrays.stream(indexDbDir.toFile().list()).parallel()
323                .filter(dir -> dir.matches("^[A-Za-z].*")
324                    && !"index.xml".equals(dir)
325                    && !"dependencies".equals(dir))
326                .map(groupId -> CompletableFuture
327                    .runAsync(() -> restoreGroup(oldGroups, groupId, true),
328                        groupLoaders))
329                .toArray(CompletableFuture[]::new));
330    }
331
332    private CompletableFuture<Void>
333            scanDependencies(Map<String, MavenGroupRepository> oldGroups) {
334        if (!depsDir.toFile().canRead() || !depsDir.toFile().isDirectory()) {
335            return CompletableFuture.completedFuture(null);
336        }
337        return CompletableFuture.allOf(
338            Arrays.stream(depsDir.toFile().list()).parallel()
339                .filter(dir -> dir.matches("^[A-Za-z].*"))
340                .map(groupId -> CompletableFuture
341                    .runAsync(() -> restoreGroup(oldGroups, groupId, false),
342                        groupLoaders))
343                .toArray(CompletableFuture[]::new));
344    }
345
346    private void restoreGroup(Map<String, MavenGroupRepository> oldGroups,
347            String groupId, boolean requested) {
348        if (oldGroups.containsKey(groupId)) {
349            // Reuse existing.
350            MavenGroupRepository groupRepo = oldGroups.get(groupId);
351            groupRepo.reset((requested ? indexDbDir : depsDir).resolve(groupId),
352                requested);
353            groups.put(groupId, groupRepo);
354            return;
355        }
356        try {
357            MavenGroupRepository groupRepo
358                = new MavenGroupRepository(groupId,
359                    (requested ? indexDbDir : depsDir).resolve(groupId),
360                    requested, this, client, reporter);
361            groups.put(groupId, groupRepo);
362        } catch (IOException e) {
363            reporter.exception(e,
364                "Cannot create group repository for %s: %s",
365                groupId, e.getMessage());
366        }
367    }
368
369    private void writeFederatedIndex(OutputStream out) {
370        DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
371        dbf.setNamespaceAware(true);
372        Document doc;
373        try {
374            doc = dbf.newDocumentBuilder().newDocument();
375        } catch (ParserConfigurationException e) {
376            return;
377        }
378        @SuppressWarnings("PMD.AvoidFinalLocalVariable")
379        final String repoNs = "http://www.osgi.org/xmlns/repository/v1.0.0";
380        // <repository name=... increment=...>
381        Element repoNode = doc.createElementNS(repoNs, "repository");
382        repoNode.setAttribute("name", name);
383        repoNode.setAttribute("increment",
384            Long.toString(System.currentTimeMillis()));
385        doc.appendChild(repoNode);
386        for (Map.Entry<String, MavenGroupRepository> repo : groups.entrySet()) {
387            // <referral url=...>
388            Element referral = doc.createElementNS(repoNs, "referral");
389            referral.setAttribute("url",
390                (repo.getValue().isRequested() ? "" : "dependencies/")
391                    + repo.getKey() + "/index.xml");
392            repoNode.appendChild(referral);
393        }
394        // Write federated index.
395        try {
396            Transformer transformer
397                = TransformerFactory.newInstance().newTransformer();
398            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
399            DOMSource source = new DOMSource(doc);
400            transformer.transform(source, new StreamResult(out));
401        } catch (TransformerException e) {
402            reporter.exception(e, "Cannot write federated index: %s",
403                e.getMessage());
404        }
405    }
406
407}