diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java index a8a80356692..ca4c93f551c 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java @@ -492,6 +492,8 @@ public void store(History history, Repository repository, String tillRevision) t * hash map entry for the file) in a file. Skip renamed files * which will be handled separately below. */ + LOGGER.log(Level.FINE, "Storing history for {0} files in repository ''{1}''", + new Object[]{map.entrySet().size(), repository.getDirectoryName()}); final File root = env.getSourceRootFile(); int fileHistoryCount = 0; for (Map.Entry> map_entry : map.entrySet()) { @@ -535,6 +537,8 @@ public void storeRenamed(Set renamedFiles, Repository repository, String renamedFiles = renamedFiles.stream().filter(f -> new File(env.getSourceRootPath() + f).exists()). collect(Collectors.toSet()); + LOGGER.log(Level.FINE, "Storing history for {0} renamed files in repository ''{1}''", + new Object[]{renamedFiles.size(), repository.getDirectoryName()}); // The directories for the renamed files have to be created before // the actual files otherwise storeFile() might be racing for diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java index 94549671073..cd0cfe8a84a 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParser.java @@ -33,6 +33,7 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Date; +import java.util.Iterator; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -68,15 +69,16 @@ class MercurialHistoryParser implements Executor.StreamHandler { * specified one. * * @param file the file or directory to get history for - * @param changeset the changeset right before the first one to fetch, or + * @param sinceRevision the changeset right before the first one to fetch, or * {@code null} if all changesets should be fetched + * @param tillRevision end revision or {@code null} * @return history for the specified file or directory * @throws HistoryException if an error happens when parsing the history */ - History parse(File file, String changeset) throws HistoryException { + History parse(File file, String sinceRevision, String tillRevision) throws HistoryException { isDir = file.isDirectory(); try { - Executor executor = repository.getHistoryLogExecutor(file, changeset); + Executor executor = repository.getHistoryLogExecutor(file, sinceRevision, tillRevision, false); int status = executor.exec(true, this); if (status != 0) { @@ -93,13 +95,28 @@ History parse(File file, String changeset) throws HistoryException { // from the list, since only the ones following it should be returned. // Also check that the specified changeset was found, otherwise throw // an exception. - if (changeset != null) { - repository.removeAndVerifyOldestChangeset(entries, changeset); + if (sinceRevision != null) { + repository.removeAndVerifyOldestChangeset(entries, sinceRevision); + } + + // See getHistoryLogExecutor() for explanation. + if (repository.isHandleRenamedFiles() && file.isFile() && tillRevision != null) { + removeChangesets(entries, tillRevision); } return new History(entries, renamedFiles); } + private void removeChangesets(List entries, String tillRevision) { + for (Iterator iter = entries.listIterator(); iter.hasNext(); ) { + HistoryEntry entry = iter.next(); + if (entry.getRevision().equals(tillRevision)) { + break; + } + iter.remove(); + } + } + /** * Process the output from the {@code hg log} command and collect * {@link HistoryEntry} elements. diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java new file mode 100644 index 00000000000..f2bd2390c36 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialHistoryParserRevisionsOnly.java @@ -0,0 +1,68 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + */ +package org.opengrok.indexer.history; + +import org.opengrok.indexer.util.Executor; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.function.Consumer; + +class MercurialHistoryParserRevisionsOnly implements Executor.StreamHandler { + private final MercurialRepository repository; + private final Consumer visitor; + + MercurialHistoryParserRevisionsOnly(MercurialRepository repository, Consumer visitor) { + this.repository = repository; + this.visitor = visitor; + } + + void parse(File file, String sinceRevision) throws HistoryException { + try { + Executor executor = repository.getHistoryLogExecutor(file, sinceRevision, null, true); + int status = executor.exec(true, this); + + if (status != 0) { + throw new HistoryException( + String.format("Failed to get revisions for: \"%s\" since revision %s Exit code: %d", + file.getAbsolutePath(), sinceRevision, status)); + } + } catch (IOException e) { + throw new HistoryException("Failed to get history for: \"" + + file.getAbsolutePath() + "\"", e); + } + } + + @Override + public void processStream(InputStream input) throws IOException { + try (BufferedReader in = new BufferedReader(new InputStreamReader(input))) { + String s; + while ((s = in.readLine()) != null) { + visitor.accept(s); + } + } + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java index 1060433f152..eb61314d29a 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java @@ -33,6 +33,7 @@ import java.util.HashMap; import java.util.List; import java.util.TreeSet; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -50,12 +51,14 @@ * Access to a Mercurial repository. * */ -public class MercurialRepository extends Repository { +public class MercurialRepository extends RepositoryWithPerPartesHistory { private static final Logger LOGGER = LoggerFactory.getLogger(MercurialRepository.class); private static final long serialVersionUID = 1L; + public static final int MAX_CHANGESETS = 256; + /** * The property name used to obtain the client command for this repository. */ @@ -81,8 +84,9 @@ public class MercurialRepository extends Repository { static final String END_OF_ENTRY = "mercurial_history_end_of_entry"; + private static final String TEMPLATE_REVS = "{rev}:{node|short}\\n"; private static final String TEMPLATE_STUB - = CHANGESET + "{rev}:{node|short}\\n" + = CHANGESET + TEMPLATE_REVS + USER + "{author}\\n" + DATE + "{date|isodate}\\n" + DESCRIPTION + "{desc|strip|obfuscate}\\n"; @@ -143,6 +147,19 @@ String determineBranch(CommandTimeoutType cmdType) throws IOException { return executor.getOutputString().trim(); } + public int getPerPartesCount() { + return MAX_CHANGESETS; + } + + private String getRevisionNum(String changeset) throws HistoryException { + String[] parts = changeset.split(":"); + if (parts.length == 2) { + return parts[0]; + } else { + throw new HistoryException("Don't know how to parse changeset identifier: " + changeset); + } + } + /** * Get an executor to be used for retrieving the history log for the named * file or directory. @@ -151,12 +168,14 @@ String determineBranch(CommandTimeoutType cmdType) throws IOException { * @param sinceRevision the oldest changeset to return from the executor, or * {@code null} if all changesets should be returned. * For files this does not apply and full history is returned. + * @param tillRevision end revision + * @param revisionsOnly get only revision numbers * @return An Executor ready to be started */ - Executor getHistoryLogExecutor(File file, String sinceRevision) + Executor getHistoryLogExecutor(File file, String sinceRevision, String tillRevision, boolean revisionsOnly) throws HistoryException, IOException { + String filename = getRepoRelativePath(file); - RuntimeEnvironment env = RuntimeEnvironment.getInstance(); List cmd = new ArrayList<>(); ensureCommand(CMD_PROPERTY_KEY, CMD_FALLBACK); @@ -164,18 +183,28 @@ Executor getHistoryLogExecutor(File file, String sinceRevision) cmd.add("log"); if (file.isDirectory()) { - // If this is non-default branch we would like to get the changesets - // on that branch and also follow any changesets from the parent branch. - if (sinceRevision != null) { + // Note: assumes one of them is not null + if ((sinceRevision != null) || (tillRevision != null)) { cmd.add("-r"); - String[] parts = sinceRevision.split(":"); - if (parts.length == 2) { - cmd.add("reverse(" + parts[0] + "::'" + getBranch() + "')"); + StringBuilder stringBuilder = new StringBuilder(); + if (!revisionsOnly) { + stringBuilder.append("reverse("); + } + if (sinceRevision != null) { + stringBuilder.append(getRevisionNum(sinceRevision)); + } + stringBuilder.append("::"); + if (tillRevision != null) { + stringBuilder.append(getRevisionNum(tillRevision)); } else { - throw new HistoryException( - "Don't know how to parse changeset identifier: " - + sinceRevision); + // If this is non-default branch we would like to get the changesets + // on that branch and also follow any changesets from the parent branch. + stringBuilder.append("'").append(getBranch()).append("'"); + } + if (!revisionsOnly) { + stringBuilder.append(")"); } + cmd.add(stringBuilder.toString()); } else { cmd.add("-r"); cmd.add("reverse(0::'" + getBranch() + "')"); @@ -191,16 +220,27 @@ Executor getHistoryLogExecutor(File file, String sinceRevision) // For files this does not matter since if getHistory() is called // for a file, the file has to be renamed so we want its complete history // if renamed file handling is enabled for this repository. + // + // Getting history for individual files should only be done when generating history for renamed files + // so the fact that filtering on sinceRevision does not work does not matter there as history + // from the initial changeset is needed. The tillRevision filtering works however not + // in combination with --follow so the filtering is done in MercurialHistoryParser.parse(). + // Even if the revision filtering worked, this approach would be probably faster and consumed less memory. if (this.isHandleRenamedFiles()) { + // When using --follow, the returned revisions are from newest to oldest, hence no reverse() is needed. cmd.add("--follow"); } } cmd.add("--template"); - if (file.isDirectory()) { - cmd.add(this.isHandleRenamedFiles() ? DIR_TEMPLATE_RENAMED : DIR_TEMPLATE); + if (revisionsOnly) { + cmd.add(TEMPLATE_REVS); } else { - cmd.add(FILE_TEMPLATE); + if (file.isDirectory()) { + cmd.add(this.isHandleRenamedFiles() ? DIR_TEMPLATE_RENAMED : DIR_TEMPLATE); + } else { + cmd.add(FILE_TEMPLATE); + } } if (!filename.isEmpty()) { @@ -259,7 +299,7 @@ private HistoryRevResult getHistoryRev( * @param fullpath file path * @param full_rev_to_find revision number (in the form of * {rev}:{node|short}) - * @returns original filename + * @return original filename */ private String findOriginalName(String fullpath, String full_rev_to_find) throws IOException { @@ -511,9 +551,17 @@ History getHistory(File file) throws HistoryException { return getHistory(file, null); } + public void accept(String sinceRevision, Consumer visitor) throws HistoryException { + new MercurialHistoryParserRevisionsOnly(this, visitor). + parse(new File(getDirectoryName()), sinceRevision); + } + + History getHistory(File file, String sinceRevision) throws HistoryException { + return getHistory(file, sinceRevision, null); + } + @Override - History getHistory(File file, String sinceRevision) - throws HistoryException { + History getHistory(File file, String sinceRevision, String tillRevision) throws HistoryException { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); // Note that the filtering of revisions based on sinceRevision is done // in the history log executor by passing appropriate options to @@ -522,9 +570,8 @@ History getHistory(File file, String sinceRevision) // for file, the file is renamed and its complete history is fetched // so no sinceRevision filter is needed. // See findOriginalName() code for more details. - History result = new MercurialHistoryParser(this).parse(file, - sinceRevision); - + History result = new MercurialHistoryParser(this).parse(file, sinceRevision, tillRevision); + // Assign tags to changesets they represent. // We don't need to check if this repository supports tags, // because we know it :-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java index 2fcbd5cd2f7..2c4dc0616c0 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java @@ -186,8 +186,7 @@ History getHistory(File file, String sinceRevision) throws HistoryException { void removeAndVerifyOldestChangeset(List entries, String revision) throws HistoryException { - HistoryEntry entry - = entries.isEmpty() ? null : entries.remove(entries.size() - 1); + HistoryEntry entry = entries.isEmpty() ? null : entries.remove(entries.size() - 1); // TODO We should check more thoroughly that the changeset is the one // we expected it to be, since some SCMs may change the revision diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java index 59f881cb85f..7813c1d80d9 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/history/MercurialRepositoryTest.java @@ -24,8 +24,10 @@ package org.opengrok.indexer.history; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.opengrok.indexer.condition.EnabledForRepository; +import org.opengrok.indexer.configuration.RuntimeEnvironment; import org.opengrok.indexer.util.Executor; import org.opengrok.indexer.util.TestRepository; @@ -37,11 +39,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opengrok.indexer.condition.RepositoryInstalled.Type.MERCURIAL; /** @@ -85,6 +89,11 @@ private void setUpTestRepository() throws IOException { repository.create(getClass().getResourceAsStream("repositories.zip")); } + @BeforeEach + public void setup() throws IOException { + setUpTestRepository(); + } + @AfterEach public void tearDown() { if (repository != null) { @@ -95,10 +104,8 @@ public void tearDown() { @Test public void testGetHistory() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); History hist = mr.getHistory(root); List entries = hist.getHistoryEntries(); assertEquals(REVISIONS.length, entries.size()); @@ -114,15 +121,13 @@ public void testGetHistory() throws Exception { @Test public void testGetHistorySubdir() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); // Add a subdirectory with some history. runHgCommand(root, "import", Paths.get(getClass().getResource("/history/hg-export-subdir.txt").toURI()).toString()); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); History hist = mr.getHistory(new File(root, "subdir")); List entries = hist.getHistoryEntries(); assertEquals(1, entries.size()); @@ -135,10 +140,8 @@ public void testGetHistorySubdir() throws Exception { */ @Test public void testGetHistoryPartial() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); // Get all but the oldest revision. History hist = mr.getHistory(root, REVISIONS[REVISIONS.length - 1]); List entries = hist.getHistoryEntries(); @@ -180,7 +183,6 @@ public static void runHgCommand(File reposRoot, String... args) { */ @Test public void testGetHistoryBranch() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); // Branch the repo and add one changeset. @@ -228,10 +230,8 @@ public void testGetHistoryBranch() throws Exception { */ @Test public void testGetHistoryGet() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); String exp_str = "This will be a first novel of mine.\n" + "\n" + "Chapter 1.\n" @@ -260,10 +260,8 @@ public void testGetHistoryGet() throws Exception { */ @Test public void testgetHistoryGetForAll() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); for (String rev : REVISIONS_novel) { InputStream input = mr.getHistoryGet(root.getCanonicalPath(), @@ -279,10 +277,8 @@ public void testgetHistoryGetForAll() throws Exception { */ @Test public void testGetHistoryGetRenamed() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); - MercurialRepository mr - = (MercurialRepository) RepositoryFactory.getRepository(root); + MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); String exp_str = "This is totally plaintext file.\n"; byte[] buffer = new byte[1024]; @@ -306,7 +302,6 @@ public void testGetHistoryGetRenamed() throws Exception { */ @Test public void testGetHistoryWithNoSuchRevision() throws Exception { - setUpTestRepository(); File root = new File(repository.getSourceRoot(), "mercurial"); MercurialRepository mr = (MercurialRepository) RepositoryFactory.getRepository(root); @@ -320,4 +315,72 @@ public void testGetHistoryWithNoSuchRevision() throws Exception { String constructedRevision = (number + 1) + ":" + hash; assertThrows(HistoryException.class, () -> mr.getHistory(root, constructedRevision)); } + + @Test + void testGetHistorySinceTillNullNull() throws Exception { + File root = new File(repository.getSourceRoot(), "mercurial"); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(root, null, null); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(10, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(REVISIONS), revisions); + } + + @Test + void testGetHistorySinceTillNullRev() throws Exception { + File root = new File(repository.getSourceRoot(), "mercurial"); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(root, null, REVISIONS[4]); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(6, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 4, REVISIONS.length)), revisions); + } + + @Test + void testGetHistorySinceTillRevNull() throws Exception { + File root = new File(repository.getSourceRoot(), "mercurial"); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(root, REVISIONS[3], null); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(3, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 0, 3)), revisions); + } + + @Test + void testGetHistorySinceTillRevRev() throws Exception { + File root = new File(repository.getSourceRoot(), "mercurial"); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(root, REVISIONS[7], REVISIONS[2]); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(5, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 2, 7)), revisions); + } + + @Test + void testGetHistoryRenamedFileTillRev() throws Exception { + RuntimeEnvironment.getInstance().setHandleHistoryOfRenamedFiles(true); + File root = new File(repository.getSourceRoot(), "mercurial"); + File file = new File(root, "novel.txt"); + assertTrue(file.exists() && file.isFile()); + MercurialRepository hgRepo = (MercurialRepository) RepositoryFactory.getRepository(root); + History history = hgRepo.getHistory(file, null, "7:db1394c05268"); + assertNotNull(history); + assertNotNull(history.getHistoryEntries()); + assertEquals(5, history.getHistoryEntries().size()); + List revisions = history.getHistoryEntries().stream().map(HistoryEntry::getRevision). + collect(Collectors.toList()); + assertEquals(List.of(Arrays.copyOfRange(REVISIONS, 2, 7)), revisions); + } }