001/*
002 * Bnd Nexus Search Plugin
003 * Copyright (C) 2017  Michael N. Lipp
004 * 
005 * This program is free software; you can redistribute it and/or modify it 
006 * under the terms of the GNU Affero General Public License as published by 
007 * the Free Software Foundation; either version 3 of the License, or 
008 * (at your option) any later version.
009 * 
010 * This program is distributed in the hope that it will be useful, but 
011 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
012 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License 
013 * for more details.
014 * 
015 * You should have received a copy of the GNU Affero General Public License along 
016 * with this program; if not, see <http://www.gnu.org/licenses/>.
017 */
018
019package de.mnl.osgi.bnd.repository.maven.nexussearch;
020
021import aQute.bnd.http.HttpClient;
022import aQute.bnd.osgi.Processor;
023import aQute.bnd.osgi.repository.XMLResourceParser;
024import aQute.lib.strings.Strings;
025import aQute.maven.api.IMavenRepo;
026import aQute.maven.api.Revision;
027import aQute.maven.provider.MavenBackingRepository;
028import aQute.maven.provider.MavenRepository;
029import aQute.service.reporter.Reporter;
030import de.mnl.osgi.bnd.repository.maven.nexussearch.NexusSearchNGResponseParser.ParseResult;
031
032import java.io.File;
033import java.io.FileInputStream;
034import java.io.FileOutputStream;
035import java.io.InputStream;
036import java.net.URL;
037import java.util.ArrayList;
038import java.util.HashMap;
039import java.util.HashSet;
040import java.util.List;
041import java.util.Map;
042import java.util.Set;
043import java.util.concurrent.Callable;
044import java.util.concurrent.ExecutorCompletionService;
045import java.util.concurrent.Executors;
046
047import javax.xml.stream.XMLEventReader;
048import javax.xml.stream.XMLInputFactory;
049import javax.xml.stream.XMLOutputFactory;
050import javax.xml.stream.XMLStreamConstants;
051import javax.xml.stream.XMLStreamWriter;
052import javax.xml.stream.events.XMLEvent;
053
054import org.osgi.resource.Resource;
055import org.osgi.service.repository.Repository;
056import org.slf4j.Logger;
057import org.slf4j.LoggerFactory;
058
059/**
060 * Provide an OSGi repository (a collection of {@link Resource}s, see 
061 * {@link Repository}), filled with the results of performing a search 
062 * on a Nexus server.
063 */
064public class NexusSearchOsgiRepository extends LocalMavenBackedOsgiRepository {
065
066        private static final Logger logger = LoggerFactory.getLogger(
067                        NexusSearchOsgiRepository.class);
068        private URL server;
069        private String queryString;
070        private int searchBreadth;
071        private int chunkSize;
072        private File localRepo = null;
073        private File mvnReposFile;
074        private Reporter reporter;
075        private HttpClient client; 
076        private MavenRepository mavenRepository;
077        
078        /**
079         * Create a new instance that uses the provided information/resources to perform
080         * its work.
081         *
082         * @param name the name
083         * @param server the url of the Nexus server
084         * @param localRepo the local Maven repository (cache)
085         * @param obrIndexFile the persistent representation of this repository's content
086         * @param mvnResposFile the mvn respos file
087         * @param queryString the query string
088         * @param searchBreadth the search breadth
089         * @param chunkSize the chunk size
090         * @param reporter a reporter for reporting the progress
091         * @param client an HTTP client for obtaining information from the Nexus server
092         * @throws Exception if a problem occurs
093         */
094        public NexusSearchOsgiRepository (String name, URL server, File localRepo, 
095                        File obrIndexFile, File mvnResposFile, String queryString, int searchBreadth,
096                        int chunkSize, Reporter reporter, HttpClient client) throws Exception {
097                super(name, obrIndexFile);
098                this.server = server;
099                this.queryString = queryString;
100                this.searchBreadth = searchBreadth;
101                this.chunkSize = chunkSize;
102                this.localRepo = localRepo;
103                this.mvnReposFile = mvnResposFile;
104                this.reporter = reporter;
105                this.client = client;
106                
107                // load results from previous execution.
108                mavenRepository = restoreRepository();
109                if (mavenRepository == null 
110                                || !location().exists() || !location().isFile()) {
111                        refresh();
112                } else {
113                        try (XMLResourceParser parser = new XMLResourceParser(location())) {
114                                List<Resource> resources = parser.parse();
115                                addAll(resources);
116                        }
117                }
118        }
119
120        private MavenRepository restoreRepository() throws Exception {
121                if (!mvnReposFile.exists()) {
122                        return null;
123                }
124                List<MavenBackingRepository> releaseBackers = new ArrayList<>();
125                List<MavenBackingRepository> snapshotBackers = new ArrayList<>();
126                List<MavenBackingRepository> backers = null;
127                XMLEventReader xmlIn = XMLInputFactory.newFactory().createXMLEventReader(
128                                new FileInputStream(mvnReposFile));
129                while (xmlIn.hasNext()) {
130                        XMLEvent event = xmlIn.nextEvent();
131                        if (event.getEventType() != XMLStreamConstants.START_ELEMENT) {
132                                continue;
133                        }
134                        switch(event.asStartElement().getName().getLocalPart()) {
135                        case "releaseUrls":
136                                backers = releaseBackers;
137                                break;
138                        case "snapshotUrls":
139                                backers = snapshotBackers;
140                                break;
141                        case "url":
142                                do {
143                                        event = xmlIn.nextEvent();
144                                } while (event.getEventType() != XMLStreamConstants.CHARACTERS);
145                                backers.addAll(MavenBackingRepository.create(
146                                                event.asCharacters().getData(), reporter, localRepo, client));
147                                break;
148                        }
149                }
150                xmlIn.close();
151                return new MavenRepository(localRepo, name(), 
152                                releaseBackers, snapshotBackers, Processor.getExecutor(), reporter);
153        }
154
155        /**
156         * Refresh this repository's content.
157         * 
158         * @return true if refreshed, false if not refreshed possibly due to error
159         * @throws Exception if a problem occurs
160         */
161        public boolean refresh() throws Exception {
162                if (queryString == null) {
163                        return false;
164                }
165                NexusSearchNGResponseParser parser = new NexusSearchNGResponseParser();
166                queryRepositories(parser);
167                queryArtifacts(parser);
168                // Repository information is obtained from both querying the repositories
169                // (provides information about existing repositories) and from executing
170                // the query (provides information about actually used repositories).
171                mavenRepository = createMavenRepository(parser);
172                Set<Revision> revsFound = parser.artifacts();
173                Map<String,List<Revision>> revsByName = new HashMap<>();
174                for (Revision rev: revsFound) {
175                        revsByName.computeIfAbsent(rev.group + ":" + rev.artifact, 
176                                        k -> new ArrayList<>()).add(rev);
177                }
178                Set<Revision> filteredRevs = new HashSet<>();
179                for (List<Revision> artifactRevs: revsByName.values()) {
180                        artifactRevs.sort(new MavenRevisionComparator().reversed());
181                        filteredRevs.addAll(artifactRevs.subList(
182                                        0, Math.min(searchBreadth, artifactRevs.size())));
183                }
184                return refresh(mavenRepository, filteredRevs);
185        }
186
187        /**
188         * Obtain information about the repositories that exist on the server.
189         * The information is stored in the parser.
190         * 
191         * @param parser the parser
192         * @throws Exception
193         */
194        private void queryRepositories(NexusSearchNGResponseParser parser) 
195                        throws Exception {
196                int attempts = 0;
197                while (true) {
198                        try {
199                                logger.debug("Getting repositories");
200                                InputStream result = client.build()
201                                                .headers("User-Agent", "Bnd")
202                                                .get(InputStream.class)
203                                                .go(new URL(server, "service/local/repositories"));
204                                parser.parse(result);
205                                result.close();
206                                break;
207                        } catch (Exception e) {
208                                attempts += 1;
209                                if (attempts > 3)
210                                        throw e;
211                                Thread.sleep(1000 * attempts);
212                        }
213                }
214        }
215        
216        /**
217         * Execute the query. The result is stored in the parser.
218         * 
219         * @param parser the parser
220         * @throws Exception
221         */
222        private void queryArtifacts(NexusSearchNGResponseParser parser) 
223                        throws Exception {
224                ExecutorCompletionService<QueryResult> exeSvc 
225                        = new ExecutorCompletionService<>(Executors.newFixedThreadPool(4));
226                int executing = 0;
227                for (String query: Strings.split(queryString)) {
228                        exeSvc.submit(new ArtifactQuery(parser, query, 1, chunkSize));
229                        executing += 1;
230                }
231                while (executing > 0) {
232                        QueryResult result = exeSvc.take().get();
233                        executing -= 1;
234                        ParseResult parsed = result.parsed;
235                        if (parsed.from > 1) {
236                                continue;
237                        }
238                        int from = parsed.from;
239                        while (from + chunkSize - 1 < parsed.totalCount) {
240                                from += chunkSize;
241                                exeSvc.submit(new ArtifactQuery(parser, result.query, from,
242                                                chunkSize));
243                                executing += 1;
244                        }
245                }
246        }
247
248        /**
249         * Execute the query. The result is stored in the parser.
250         */
251        class ArtifactQuery implements Callable<QueryResult> {
252                private NexusSearchNGResponseParser parser;
253                private String query;
254                private int from;
255                private int count;
256                
257                public ArtifactQuery(NexusSearchNGResponseParser parser, String query, int from, int count) {
258                        super();
259                        this.parser = parser;
260                        this.query = query;
261                        this.from = from;
262                        this.count = count;
263                }
264
265                @Override
266                public QueryResult call() throws Exception {
267                        int attempts = 0;
268                        QueryResult result = new QueryResult();
269                        result.query = query;
270                        while (true) {
271                                try {
272                                        logger.debug("Searching {}", query);
273                                        InputStream answer = client.build()
274                                                        .headers("User-Agent", "Bnd")
275                                                        .get(InputStream.class)
276                                                        .go(new URL(server, "service/local/lucene/search?"
277                                                                        + query + "&from=" + from + "&count=" + count));
278                                        result.parsed = parser.parse(answer);
279                                        answer.close();
280                                        logger.debug("Got for {} results from {} to {} (of {})", 
281                                                        query, result.parsed.from, 
282                                                        result.parsed.from + result.parsed.count - 1,
283                                                        result.parsed.totalCount);
284                                        if (result.parsed.tooManyResults 
285                                                && result.parsed.count < count) {
286                                                logger.error("Too many results for {}, results were "
287                                                                + "lost (chunk size too big)", query);
288                                        }
289                                        break;
290                                } catch (Exception e) {
291                                        attempts += 1;
292                                        if (attempts > 3)
293                                                throw e;
294                                        Thread.sleep(1000 * attempts);
295                                }
296                        }
297
298                        // List all revisions from query.
299                        for (Revision revision: parser.artifacts()) {
300                                logger.debug("Found {}", revision);                                     
301                        }
302                
303                        return result;
304                }
305        }
306
307        private class QueryResult {
308                String query;
309                ParseResult parsed;
310        }
311        
312        private MavenRepository createMavenRepository(
313                        NexusSearchNGResponseParser parser) throws Exception {
314                // Create repository from URLs
315                XMLStreamWriter xmlOut = XMLOutputFactory.newFactory().createXMLStreamWriter(
316                                new FileOutputStream(mvnReposFile));
317                xmlOut.writeStartDocument();
318                xmlOut.writeStartElement("repositories");
319                xmlOut.writeStartElement("repository");
320                xmlOut.writeStartElement("releaseUrls");
321                List<MavenBackingRepository> releaseBackers = new ArrayList<>();
322                for (URL repoUrl: parser.releaseRepositories()) {
323                        xmlOut.writeStartElement("url");
324                        xmlOut.writeCharacters(repoUrl.toString());
325                        xmlOut.writeEndElement();
326                        releaseBackers.addAll(MavenBackingRepository.create(
327                                        repoUrl.toString(), reporter, localRepo, client));
328                }
329                xmlOut.writeEndElement();
330                xmlOut.writeStartElement("snapshotUrls");
331                List<MavenBackingRepository> snapshotBackers = new ArrayList<>();
332                for (URL repoUrl: parser.snapshotRepositories()) {
333                        xmlOut.writeStartElement("url");
334                        xmlOut.writeCharacters(repoUrl.toString());
335                        xmlOut.writeEndElement();
336                        snapshotBackers.addAll(MavenBackingRepository.create(
337                                        repoUrl.toString(), reporter, localRepo, client));
338                }
339                xmlOut.writeEndElement();
340                xmlOut.writeEndElement();
341                xmlOut.writeEndElement();
342                xmlOut.writeEndDocument();
343                xmlOut.close();
344                return new MavenRepository(localRepo, name(), 
345                                releaseBackers, snapshotBackers,
346                                Processor.getExecutor(), reporter);
347        }
348        
349        /**
350         * Return the Maven repository object used to back this repository.
351         *
352         * @return the maven repository
353         * @throws Exception if a problem occurs
354         */
355        public IMavenRepo mavenRepository() throws Exception {
356                if (mavenRepository == null) {
357                        refresh();
358                }
359                return mavenRepository;
360        }
361}