Skip to content
This repository was archived by the owner on Nov 19, 2020. It is now read-only.

Commit 63e389d

Browse files
mccullsmcculls
committed
NEXUS-8226: limit reconciliation to checksum attributes modified since a given date
1 parent ef1b443 commit 63e389d

File tree

4 files changed

+133
-25
lines changed

4 files changed

+133
-25
lines changed

‎components/nexus-core/src/main/java/org/sonatype/nexus/proxy/item/ChecksumReconciler.java‎

Lines changed: 70 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
*/
1313
package org.sonatype.nexus.proxy.item;
1414

15+
import java.io.File;
1516
import java.io.InputStream;
1617
import java.security.DigestInputStream;
1718
import java.security.MessageDigest;
@@ -26,6 +27,7 @@
2627
import org.sonatype.nexus.proxy.attributes.inspectors.DigestCalculatingInspector;
2728
import org.sonatype.nexus.proxy.maven.MUtils;
2829
import org.sonatype.nexus.proxy.repository.Repository;
30+
import org.sonatype.nexus.proxy.storage.local.fs.DefaultFSLocalRepositoryStorage;
2931
import org.sonatype.nexus.proxy.walker.AbstractFileWalkerProcessor;
3032
import org.sonatype.nexus.proxy.walker.DefaultWalkerContext;
3133
import org.sonatype.nexus.proxy.walker.Walker;
@@ -35,6 +37,14 @@
3537
import org.sonatype.sisu.goodies.common.ComponentSupport;
3638

3739
import com.google.common.io.ByteStreams;
40+
import org.apache.commons.lang.StringUtils;
41+
42+
import static org.sonatype.nexus.proxy.attributes.inspectors.DigestCalculatingInspector.DIGEST_MD5_KEY;
43+
import static org.sonatype.nexus.proxy.attributes.inspectors.DigestCalculatingInspector.DIGEST_SHA1_KEY;
44+
import static org.sonatype.nexus.proxy.maven.ChecksumContentValidator.ATTR_REMOTE_MD5;
45+
import static org.sonatype.nexus.proxy.maven.ChecksumContentValidator.ATTR_REMOTE_SHA1;
46+
import static org.sonatype.nexus.proxy.maven.ChecksumContentValidator.SUFFIX_MD5;
47+
import static org.sonatype.nexus.proxy.maven.ChecksumContentValidator.SUFFIX_SHA1;
3848

3949
/**
4050
* Reconciles any item attribute checksums affected by NEXUS-8178.
@@ -53,6 +63,10 @@ public class ChecksumReconciler
5363

5464
private final MessageDigest md5;
5565

66+
private File attributesBaseDir;
67+
68+
private long modifiedSinceMillis;
69+
5670
@Inject
5771
public ChecksumReconciler(final Walker walker, final DigestCalculatingInspector digestCalculatingInspector)
5872
throws NoSuchAlgorithmException
@@ -68,10 +82,18 @@ public ChecksumReconciler(final Walker walker, final DigestCalculatingInspector
6882
/**
6983
* Walks the selected sub-tree in the given repository attempting to reconcile their item attribute checksums.
7084
*/
71-
public void reconcileChecksums(final Repository repo, final ResourceStoreRequest request) {
85+
public void reconcileChecksums(final Repository repo, final ResourceStoreRequest request, final long sinceMillis) {
7286
final WalkerContext context = new DefaultWalkerContext(repo, request);
7387

7488
final String repositoryId = repo.getId();
89+
90+
attributesBaseDir = getAttributesBaseDir(repo);
91+
modifiedSinceMillis = sinceMillis;
92+
93+
if (attributesBaseDir == null) {
94+
return; // no point walking repository with no local storage
95+
}
96+
7597
context.getProcessors().add(new AbstractFileWalkerProcessor()
7698
{
7799
@Override
@@ -97,17 +119,21 @@ protected void processFileItem(final WalkerContext ctx, final StorageFileItem it
97119
* Checks the checksum cached in the item's attributes to see if it needs reconciling with the stored content.
98120
*/
99121
void reconcileItemChecksum(final Repository repo, final StorageFileItem item) throws Exception {
100-
final String itemPath = item.getPath();
101-
if (!itemPath.endsWith(".sha1") && !itemPath.endsWith(".md5")) {
102-
final String attributeSHA1 = item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_SHA1_KEY);
103-
if (attributeSHA1 != null) {
122+
123+
final String localSHA1 = item.getRepositoryItemAttributes().get(DIGEST_SHA1_KEY);
124+
final String remoteSHA1 = item.getRepositoryItemAttributes().get(ATTR_REMOTE_SHA1);
125+
126+
if (localSHA1 != null || remoteSHA1 != null) {
127+
128+
final String itemPath = item.getPath();
129+
if (shouldAttemptReconciliation(itemPath)) {
104130

105131
sha1.reset();
106132
md5.reset();
107133

108134
// look for checksums affected by the link persister pre-fetching the first 8 bytes (see NEXUS-8178)
109-
try (final InputStream is = new DigestInputStream(new DigestInputStream( //
110-
item.getContentLocator().getContent(), sha1), md5)) {
135+
try (final InputStream is = new DigestInputStream(new DigestInputStream(item.getContentLocator().getContent(),
136+
sha1), md5)) {
111137

112138
final byte[] buf = new byte[8];
113139
ByteStreams.read(is, buf, 0, 8);
@@ -119,11 +145,16 @@ void reconcileItemChecksum(final Repository repo, final StorageFileItem item) th
119145

120146
final String affectedSHA1 = DigesterUtils.getDigestAsString(sha1.digest());
121147

122-
if (attributeSHA1.equals(affectedSHA1)) {
148+
if ((localSHA1 != null && localSHA1.equals(affectedSHA1))
149+
|| (remoteSHA1 != null && remoteSHA1.equals(affectedSHA1))) {
150+
123151
log.info("Reconciling attribute checksums for {}", item);
152+
124153
final RepositoryItemUid uid = item.getRepositoryItemUid();
125154
uid.getLock().lock(Action.update);
126155
try {
156+
item.getRepositoryItemAttributes().remove(ATTR_REMOTE_SHA1);
157+
item.getRepositoryItemAttributes().remove(ATTR_REMOTE_MD5);
127158
digestCalculatingInspector.processStorageItem(item);
128159
repo.getAttributesHandler().storeAttributes(item);
129160
}
@@ -132,15 +163,42 @@ void reconcileItemChecksum(final Repository repo, final StorageFileItem item) th
132163
}
133164
}
134165

135-
reconcileChecksumFile(repo, itemPath + ".sha1", affectedSHA1,
136-
item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_SHA1_KEY));
166+
reconcileChecksumFile(repo, itemPath + SUFFIX_SHA1, affectedSHA1,
167+
item.getRepositoryItemAttributes().get(DIGEST_SHA1_KEY));
137168

138169
final String affectedMD5 = DigesterUtils.getDigestAsString(md5.digest());
139170

140-
reconcileChecksumFile(repo, itemPath + ".md5", affectedMD5,
141-
item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_MD5_KEY));
171+
reconcileChecksumFile(repo, itemPath + SUFFIX_MD5, affectedMD5,
172+
item.getRepositoryItemAttributes().get(DIGEST_MD5_KEY));
173+
}
174+
}
175+
}
176+
177+
/**
178+
* Determines the base directory of local attribute storage.
179+
*/
180+
private File getAttributesBaseDir(final Repository repo) {
181+
try {
182+
if (repo.getLocalStorage() instanceof DefaultFSLocalRepositoryStorage) {
183+
final File baseDir = ((DefaultFSLocalRepositoryStorage) repo.getLocalStorage()).getBaseDir(repo, null);
184+
return new File(new File(baseDir, ".nexus"), "attributes");
142185
}
143186
}
187+
catch (final Exception e) {
188+
log.warn("Problem finding local storage for {}", repo, e);
189+
}
190+
return null;
191+
}
192+
193+
/**
194+
* Should we attempt checksum reconciliation for the given path?
195+
*/
196+
private boolean shouldAttemptReconciliation(final String itemPath) {
197+
if (itemPath == null || itemPath.endsWith(SUFFIX_SHA1) || itemPath.endsWith(SUFFIX_MD5)) {
198+
return false; // ignore associated checksum files, we'll fix them when we process the main item
199+
}
200+
final File attributesFile = new File(attributesBaseDir, StringUtils.strip(itemPath, "/"));
201+
return attributesFile.lastModified() >= modifiedSinceMillis;
144202
}
145203

146204
/**

‎components/nexus-core/src/main/java/org/sonatype/nexus/tasks/ReconcileChecksumsTask.java‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,9 @@
2727
import org.sonatype.nexus.scheduling.AbstractNexusRepositoriesPathAwareTask;
2828
import org.sonatype.nexus.tasks.descriptors.ReconcileChecksumsTaskDescriptor;
2929
import org.sonatype.nexus.util.LinearNumberSequence;
30-
import org.sonatype.nexus.util.SystemPropertiesHelper;
30+
31+
import org.apache.commons.lang.StringUtils;
32+
import org.joda.time.DateTime;
3133

3234
import static com.google.common.base.Preconditions.checkNotNull;
3335

@@ -45,13 +47,6 @@ public class ReconcileChecksumsTask
4547
*/
4648
public static final String ACTION = "RECONCILECHECKSUMS";
4749

48-
/**
49-
* System property used to control throttling
50-
*/
51-
public static final String MAX_RATE_KEY = ReconcileChecksumsTask.class.getName() + ".maxRate";
52-
53-
private static final int MAX_RATE = SystemPropertiesHelper.getInteger(MAX_RATE_KEY, 100);
54-
5550
private final ChecksumReconciler checksumReconciler;
5651

5752
@Inject
@@ -69,6 +64,24 @@ protected String getRepositoryPathFieldId() {
6964
return ReconcileChecksumsTaskDescriptor.RESOURCE_STORE_PATH_FIELD_ID;
7065
}
7166

67+
public long getModifiedSinceMillis() {
68+
final String value = getParameter(ReconcileChecksumsTaskDescriptor.MODIFIED_SINCE_DATE_ID);
69+
return StringUtils.isNotBlank(value) ? DateTime.parse(value).getMillis() : -1;
70+
}
71+
72+
public void setModifiedSinceDate(final String modifiedSinceDate) {
73+
addParameter(ReconcileChecksumsTaskDescriptor.MODIFIED_SINCE_DATE_ID, modifiedSinceDate);
74+
}
75+
76+
public int getWalkingLimitTps() {
77+
final String value = getParameter(ReconcileChecksumsTaskDescriptor.WALKING_LIMIT_TPS_FIELD_ID);
78+
return StringUtils.isNotBlank(value) ? Integer.parseInt(value) : -1;
79+
}
80+
81+
public void setWalkingLimitTps(final int walkingLimitTps) {
82+
addParameter(ReconcileChecksumsTaskDescriptor.WALKING_LIMIT_TPS_FIELD_ID, Integer.toString(walkingLimitTps));
83+
}
84+
7285
@Override
7386
public Object doRun() throws Exception {
7487
final List<Repository> targetRepositories = new ArrayList<>();
@@ -95,13 +108,14 @@ public Object doRun() throws Exception {
95108

96109
final ResourceStoreRequest request = new ResourceStoreRequest(getResourceStorePath(), true, false);
97110

98-
if (MAX_RATE > 0) {
111+
final int limitTps = getWalkingLimitTps();
112+
if (limitTps > 0) {
99113
request.getRequestContext().put(WalkerThrottleController.CONTEXT_KEY,
100-
new FixedRateWalkerThrottleController(MAX_RATE, new LinearNumberSequence(0, 1, 1, 0)));
114+
new FixedRateWalkerThrottleController(limitTps, new LinearNumberSequence(0, 1, 1, 0)));
101115
}
102116

103117
for (final Repository repo : targetRepositories) {
104-
checksumReconciler.reconcileChecksums(repo, request);
118+
checksumReconciler.reconcileChecksums(repo, request, getModifiedSinceMillis());
105119
}
106120

107121
return null;

‎components/nexus-core/src/main/java/org/sonatype/nexus/tasks/descriptors/ReconcileChecksumsTaskDescriptor.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import javax.inject.Singleton;
2020

2121
import org.sonatype.nexus.formfields.FormField;
22+
import org.sonatype.nexus.formfields.NumberTextFormField;
2223
import org.sonatype.nexus.formfields.RepoOrGroupComboFormField;
2324
import org.sonatype.nexus.formfields.StringTextFormField;
2425

@@ -36,6 +37,10 @@ public class ReconcileChecksumsTaskDescriptor
3637

3738
public static final String RESOURCE_STORE_PATH_FIELD_ID = "resourceStorePath";
3839

40+
public static final String MODIFIED_SINCE_DATE_ID = "modifiedSinceDate";
41+
42+
public static final String WALKING_LIMIT_TPS_FIELD_ID = "walkingLimitTps";
43+
3944
private final RepoOrGroupComboFormField repoField = new RepoOrGroupComboFormField(REPO_OR_GROUP_FIELD_ID,
4045
FormField.MANDATORY);
4146

@@ -44,6 +49,16 @@ public class ReconcileChecksumsTaskDescriptor
4449
"Enter a repository path to run the task in recursively (ie. \"/\" for root or \"/org/apache\").",
4550
FormField.OPTIONAL);
4651

52+
private final StringTextFormField modifiedSinceDateField = new StringTextFormField(MODIFIED_SINCE_DATE_ID,
53+
"Modified since (yyyy-MM-dd)",
54+
"Enter a date to limit reconciliation to those checksum attributes modified since the given date.",
55+
FormField.OPTIONAL).withInitialValue("2015-01-01");
56+
57+
private final NumberTextFormField walkingLimitTpsField = new NumberTextFormField(WALKING_LIMIT_TPS_FIELD_ID,
58+
"Walking limit (tps)",
59+
"Set the walking limit to reduce the overhead of this task at the expense of it taking more time.",
60+
FormField.OPTIONAL).withInitialValue(100);
61+
4762
public String getId() {
4863
return ID;
4964
}
@@ -57,8 +72,9 @@ public List<FormField> formFields() {
5772
final List<FormField> fields = new ArrayList<FormField>();
5873

5974
fields.add(repoField);
60-
6175
fields.add(resourceStorePathField);
76+
fields.add(modifiedSinceDateField);
77+
fields.add(walkingLimitTpsField);
6278

6379
return fields;
6480
}

‎components/nexus-core/src/test/java/org/sonatype/nexus/tasks/ReconcileChecksumsTaskTest.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.sonatype.nexus.proxy.maven.MUtils;
2121
import org.sonatype.nexus.scheduling.NexusScheduler;
2222

23+
import org.joda.time.DateTime;
2324
import org.junit.Test;
2425

2526
import static org.hamcrest.MatcherAssert.assertThat;
@@ -94,8 +95,27 @@ public void testChecksumsAreReconciled() throws Exception {
9495
assertThat(MUtils.readDigestFromFileItem((StorageFileItem) snapshots.retrieveItem(metadataMD5)),
9596
is("ec91980b1f83604ed72d749eebacea53"));
9697

97-
task.setRepositoryId(snapshots.getId());
9898
task.setResourceStorePath("/org/sonatype/");
99+
task.setModifiedSinceDate(DateTime.now().plusYears(1).toString("yyyy-MM-dd"));
100+
101+
nexusScheduler.submit("testAffectedSubTree", task).get();
102+
103+
item = snapshots.retrieveItem(metadataXML);
104+
105+
sha1sum = item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_SHA1_KEY);
106+
md5sum = item.getRepositoryItemAttributes().get(DigestCalculatingInspector.DIGEST_MD5_KEY);
107+
108+
// checksums still need to be reconciled
109+
assertThat(sha1sum, is("18da84408c713e55a3c55dc34a1ccff78086fd46"));
110+
assertThat(md5sum, is("ec91980b1f83604ed72d749eebacea53"));
111+
112+
// associated checksum files still need to be reconciled
113+
assertThat(MUtils.readDigestFromFileItem((StorageFileItem) snapshots.retrieveItem(metadataSHA1)),
114+
is("18da84408c713e55a3c55dc34a1ccff78086fd46"));
115+
assertThat(MUtils.readDigestFromFileItem((StorageFileItem) snapshots.retrieveItem(metadataMD5)),
116+
is("ec91980b1f83604ed72d749eebacea53"));
117+
118+
task.setModifiedSinceDate("2015-01-01");
99119

100120
nexusScheduler.submit("testAffectedSubTree", task).get();
101121

0 commit comments

Comments
 (0)