Source code

001/*
002 * Bnd Nexus Search Plugin
003 * Copyright (C) 2017  Michael N. Lipp
004 * 
005 * This program is free software; you can redistribute it and/or modify it 
006 * under the terms of the GNU Affero General Public License as published by 
007 * the Free Software Foundation; either version 3 of the License, or 
008 * (at your option) any later version.
009 * 
010 * This program is distributed in the hope that it will be useful, but 
011 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
012 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License 
013 * for more details.
014 * 
015 * You should have received a copy of the GNU Affero General Public License along 
016 * with this program; if not, see <http://www.gnu.org/licenses/>.
017 */
018
019package de.mnl.osgi.bnd.repository.maven.nexussearch;
020
021import aQute.bnd.http.HttpClient;
022import aQute.bnd.osgi.Processor;
023import aQute.bnd.osgi.repository.XMLResourceParser;
024import aQute.lib.strings.Strings;
025import aQute.maven.api.IMavenRepo;
026import aQute.maven.api.Revision;
027import aQute.maven.provider.MavenBackingRepository;
028import aQute.maven.provider.MavenRepository;
029import aQute.service.reporter.Reporter;
030import de.mnl.osgi.bnd.repository.maven.nexussearch.NexusSearchNGResponseParser.ParseResult;
031import java.io.File;
032import java.io.FileInputStream;
033import java.io.FileOutputStream;
034import java.io.InputStream;
035import java.net.URL;
036import java.util.ArrayList;
037import java.util.HashMap;
038import java.util.HashSet;
039import java.util.List;
040import java.util.Map;
041import java.util.Set;
042import java.util.concurrent.Callable;
043import java.util.concurrent.ExecutorCompletionService;
044import java.util.concurrent.Executors;
045import javax.xml.stream.XMLEventReader;
046import javax.xml.stream.XMLInputFactory;
047import javax.xml.stream.XMLOutputFactory;
048import javax.xml.stream.XMLStreamConstants;
049import javax.xml.stream.XMLStreamWriter;
050import javax.xml.stream.events.XMLEvent;
051import org.osgi.resource.Resource;
052import org.osgi.service.repository.Repository;
053import org.slf4j.Logger;
054import org.slf4j.LoggerFactory;
055
056/**
057 * Provide an OSGi repository (a collection of {@link Resource}s, see 
058 * {@link Repository}), filled with the results of performing a search 
059 * on a Nexus server.
060 */
061public class NexusSearchOsgiRepository extends LocalMavenBackedOsgiRepository {
062
063    private static final Logger logger = LoggerFactory.getLogger(
064        NexusSearchOsgiRepository.class);
065    private URL server;
066    private String queryString;
067    private int searchBreadth;
068    private int chunkSize;
069    private File localRepo = null;
070    private File mvnReposFile;
071    private Reporter reporter;
072    private HttpClient client;
073    private MavenRepository mavenRepository;
074
075    /**
076     * Create a new instance that uses the provided information/resources to perform
077     * its work.
078     *
079     * @param name the name
080     * @param server the url of the Nexus server
081     * @param localRepo the local Maven repository (cache)
082     * @param obrIndexFile the persistent representation of this repository's content
083     * @param mvnResposFile the mvn respos file
084     * @param queryString the query string
085     * @param searchBreadth the search breadth
086     * @param chunkSize the chunk size
087     * @param reporter a reporter for reporting the progress
088     * @param client an HTTP client for obtaining information from the Nexus server
089     * @throws Exception if a problem occurs
090     */
091    public NexusSearchOsgiRepository(String name, URL server, File localRepo,
092            File obrIndexFile, File mvnResposFile, String queryString,
093            int searchBreadth,
094            int chunkSize, Reporter reporter, HttpClient client)
095            throws Exception {
096        super(name, obrIndexFile);
097        this.server = server;
098        this.queryString = queryString;
099        this.searchBreadth = searchBreadth;
100        this.chunkSize = chunkSize;
101        this.localRepo = localRepo;
102        this.mvnReposFile = mvnResposFile;
103        this.reporter = reporter;
104        this.client = client;
105
106        // load results from previous execution.
107        mavenRepository = restoreRepository();
108        if (mavenRepository == null
109            || !location().exists() || !location().isFile()) {
110            refresh();
111        } else {
112            try (XMLResourceParser parser = new XMLResourceParser(location())) {
113                List<Resource> resources = parser.parse();
114                addAll(resources);
115            }
116        }
117    }
118
119    private MavenRepository restoreRepository() throws Exception {
120        if (!mvnReposFile.exists()) {
121            return null;
122        }
123        List<MavenBackingRepository> releaseBackers = new ArrayList<>();
124        List<MavenBackingRepository> snapshotBackers = new ArrayList<>();
125        List<MavenBackingRepository> backers = null;
126        XMLEventReader xmlIn
127            = XMLInputFactory.newFactory().createXMLEventReader(
128                new FileInputStream(mvnReposFile));
129        while (xmlIn.hasNext()) {
130            XMLEvent event = xmlIn.nextEvent();
131            if (event.getEventType() != XMLStreamConstants.START_ELEMENT) {
132                continue;
133            }
134            switch (event.asStartElement().getName().getLocalPart()) {
135            case "releaseUrls":
136                backers = releaseBackers;
137                break;
138            case "snapshotUrls":
139                backers = snapshotBackers;
140                break;
141            case "url":
142                do {
143                    event = xmlIn.nextEvent();
144                } while (event.getEventType() != XMLStreamConstants.CHARACTERS);
145                backers.addAll(MavenBackingRepository.create(
146                    event.asCharacters().getData(), reporter, localRepo,
147                    client));
148                break;
149            }
150        }
151        xmlIn.close();
152        return new MavenRepository(localRepo, name(),
153            releaseBackers, snapshotBackers, Processor.getExecutor(), reporter);
154    }
155
156    /**
157     * Refresh this repository's content.
158     * 
159     * @return true if refreshed, false if not refreshed possibly due to error
160     * @throws Exception if a problem occurs
161     */
162    public boolean refresh() throws Exception {
163        if (queryString == null) {
164            return false;
165        }
166        NexusSearchNGResponseParser parser = new NexusSearchNGResponseParser();
167        queryRepositories(parser);
168        queryArtifacts(parser);
169        // Repository information is obtained from both querying the
170        // repositories
171        // (provides information about existing repositories) and from executing
172        // the query (provides information about actually used repositories).
173        mavenRepository = createMavenRepository(parser);
174        Set<Revision> revsFound = parser.artifacts();
175        Map<String, List<Revision>> revsByName = new HashMap<>();
176        for (Revision rev : revsFound) {
177            revsByName.computeIfAbsent(rev.group + ":" + rev.artifact,
178                k -> new ArrayList<>()).add(rev);
179        }
180        Set<Revision> filteredRevs = new HashSet<>();
181        for (List<Revision> artifactRevs : revsByName.values()) {
182            artifactRevs.sort(new MavenRevisionComparator().reversed());
183            filteredRevs.addAll(artifactRevs.subList(
184                0, Math.min(searchBreadth, artifactRevs.size())));
185        }
186        return refresh(mavenRepository, filteredRevs);
187    }
188
189    /**
190     * Obtain information about the repositories that exist on the server.
191     * The information is stored in the parser.
192     * 
193     * @param parser the parser
194     * @throws Exception
195     */
196    private void queryRepositories(NexusSearchNGResponseParser parser)
197            throws Exception {
198        int attempts = 0;
199        while (true) {
200            try {
201                logger.debug("Getting repositories");
202                InputStream result = client.build()
203                    .headers("User-Agent", "Bnd")
204                    .get(InputStream.class)
205                    .go(new URL(server, "service/local/repositories"));
206                parser.parse(result);
207                result.close();
208                break;
209            } catch (Exception e) {
210                attempts += 1;
211                if (attempts > 3) {
212                    throw e;
213                }
214                Thread.sleep(1000 * attempts);
215            }
216        }
217    }
218
219    /**
220     * Execute the query. The result is stored in the parser.
221     * 
222     * @param parser the parser
223     * @throws Exception
224     */
225    private void queryArtifacts(NexusSearchNGResponseParser parser)
226            throws Exception {
227        ExecutorCompletionService<QueryResult> exeSvc
228            = new ExecutorCompletionService<>(Executors.newFixedThreadPool(4));
229        int executing = 0;
230        for (String query : Strings.split(queryString)) {
231            exeSvc.submit(new ArtifactQuery(parser, query, 1, chunkSize));
232            executing += 1;
233        }
234        while (executing > 0) {
235            QueryResult result = exeSvc.take().get();
236            executing -= 1;
237            ParseResult parsed = result.parsed;
238            if (parsed.from > 1) {
239                continue;
240            }
241            int from = parsed.from;
242            while (from + chunkSize - 1 < parsed.totalCount) {
243                from += chunkSize;
244                exeSvc.submit(new ArtifactQuery(parser, result.query, from,
245                    chunkSize));
246                executing += 1;
247            }
248        }
249    }
250
251    /**
252     * Execute the query. The result is stored in the parser.
253     */
254    class ArtifactQuery implements Callable<QueryResult> {
255        private NexusSearchNGResponseParser parser;
256        private String query;
257        private int from;
258        private int count;
259
260        public ArtifactQuery(NexusSearchNGResponseParser parser, String query,
261                int from, int count) {
262            super();
263            this.parser = parser;
264            this.query = query;
265            this.from = from;
266            this.count = count;
267        }
268
269        @Override
270        public QueryResult call() throws Exception {
271            int attempts = 0;
272            QueryResult result = new QueryResult();
273            result.query = query;
274            while (true) {
275                try {
276                    logger.debug("Searching {}", query);
277                    InputStream answer = client.build()
278                        .headers("User-Agent", "Bnd")
279                        .get(InputStream.class)
280                        .go(new URL(server, "service/local/lucene/search?"
281                            + query + "&from=" + from + "&count=" + count));
282                    result.parsed = parser.parse(answer);
283                    answer.close();
284                    logger.debug("Got for {} results from {} to {} (of {})",
285                        query, result.parsed.from,
286                        result.parsed.from + result.parsed.count - 1,
287                        result.parsed.totalCount);
288                    if (result.parsed.tooManyResults
289                        && result.parsed.count < count) {
290                        logger.error("Too many results for {}, results were "
291                            + "lost (chunk size too big)", query);
292                    }
293                    break;
294                } catch (Exception e) {
295                    attempts += 1;
296                    if (attempts > 3) {
297                        throw e;
298                    }
299                    Thread.sleep(1000 * attempts);
300                }
301            }
302
303            // List all revisions from query.
304            for (Revision revision : parser.artifacts()) {
305                logger.debug("Found {}", revision);
306            }
307
308            return result;
309        }
310    }
311
312    private class QueryResult {
313        String query;
314        ParseResult parsed;
315    }
316
317    private MavenRepository createMavenRepository(
318            NexusSearchNGResponseParser parser) throws Exception {
319        // Create repository from URLs
320        XMLStreamWriter xmlOut
321            = XMLOutputFactory.newFactory().createXMLStreamWriter(
322                new FileOutputStream(mvnReposFile));
323        xmlOut.writeStartDocument();
324        xmlOut.writeStartElement("repositories");
325        xmlOut.writeStartElement("repository");
326        xmlOut.writeStartElement("releaseUrls");
327        List<MavenBackingRepository> releaseBackers = new ArrayList<>();
328        for (URL repoUrl : parser.releaseRepositories()) {
329            xmlOut.writeStartElement("url");
330            xmlOut.writeCharacters(repoUrl.toString());
331            xmlOut.writeEndElement();
332            releaseBackers.addAll(MavenBackingRepository.create(
333                repoUrl.toString(), reporter, localRepo, client));
334        }
335        xmlOut.writeEndElement();
336        xmlOut.writeStartElement("snapshotUrls");
337        List<MavenBackingRepository> snapshotBackers = new ArrayList<>();
338        for (URL repoUrl : parser.snapshotRepositories()) {
339            xmlOut.writeStartElement("url");
340            xmlOut.writeCharacters(repoUrl.toString());
341            xmlOut.writeEndElement();
342            snapshotBackers.addAll(MavenBackingRepository.create(
343                repoUrl.toString(), reporter, localRepo, client));
344        }
345        xmlOut.writeEndElement();
346        xmlOut.writeEndElement();
347        xmlOut.writeEndElement();
348        xmlOut.writeEndDocument();
349        xmlOut.close();
350        return new MavenRepository(localRepo, name(),
351            releaseBackers, snapshotBackers,
352            Processor.getExecutor(), reporter);
353    }
354
355    /**
356     * Return the Maven repository object used to back this repository.
357     *
358     * @return the maven repository
359     * @throws Exception if a problem occurs
360     */
361    public IMavenRepo mavenRepository() throws Exception {
362        if (mavenRepository == null) {
363            refresh();
364        }
365        return mavenRepository;
366    }
367}