Skip to content

Commit a59599d

Browse files
ajantha-bhatajantha-bhat
authored
Nessie: Refactor NessieTableOperations#doCommit (#6240)
* Nessie: Refactor NessieTableOperations * Expose refName
1 parent b156fff commit a59599d

File tree

2 files changed

+93
-82
lines changed

2 files changed

+93
-82
lines changed

‎nessie/src/main/java/org/apache/iceberg/nessie/NessieIcebergClient.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
import java.util.function.Predicate;
2626
import java.util.function.Supplier;
2727
import java.util.stream.Collectors;
28+
import org.apache.iceberg.Snapshot;
29+
import org.apache.iceberg.TableMetadata;
2830
import org.apache.iceberg.catalog.Namespace;
2931
import org.apache.iceberg.catalog.TableIdentifier;
3032
import org.apache.iceberg.exceptions.AlreadyExistsException;
@@ -52,6 +54,8 @@
5254
import org.projectnessie.model.EntriesResponse;
5355
import org.projectnessie.model.GetNamespacesResponse;
5456
import org.projectnessie.model.IcebergTable;
57+
import org.projectnessie.model.ImmutableCommitMeta;
58+
import org.projectnessie.model.ImmutableIcebergTable;
5559
import org.projectnessie.model.Operation;
5660
import org.projectnessie.model.Reference;
5761
import org.projectnessie.model.Tag;
@@ -416,6 +420,91 @@ public boolean dropTable(TableIdentifier identifier, boolean purge) {
416420
return !threw;
417421
}
418422

423+
public void commitTable(
424+
TableMetadata base,
425+
TableMetadata metadata,
426+
String newMetadataLocation,
427+
IcebergTable expectedContent,
428+
ContentKey key)
429+
throws NessieConflictException, NessieNotFoundException {
430+
UpdateableReference updateableReference = getRef();
431+
432+
updateableReference.checkMutable();
433+
434+
Branch current = updateableReference.getAsBranch();
435+
Branch expectedHead = current;
436+
if (base != null) {
437+
String metadataCommitId =
438+
base.property(NessieTableOperations.NESSIE_COMMIT_ID_PROPERTY, expectedHead.getHash());
439+
if (metadataCommitId != null) {
440+
expectedHead = Branch.of(expectedHead.getName(), metadataCommitId);
441+
}
442+
}
443+
444+
ImmutableIcebergTable.Builder newTableBuilder = ImmutableIcebergTable.builder();
445+
if (expectedContent != null) {
446+
newTableBuilder.id(expectedContent.getId());
447+
}
448+
Snapshot snapshot = metadata.currentSnapshot();
449+
long snapshotId = snapshot != null ? snapshot.snapshotId() : -1L;
450+
451+
IcebergTable newTable =
452+
newTableBuilder
453+
.snapshotId(snapshotId)
454+
.schemaId(metadata.currentSchemaId())
455+
.specId(metadata.defaultSpecId())
456+
.sortOrderId(metadata.defaultSortOrderId())
457+
.metadataLocation(newMetadataLocation)
458+
.build();
459+
460+
LOG.debug(
461+
"Committing '{}' against '{}', current is '{}': {}",
462+
key,
463+
expectedHead,
464+
current.getHash(),
465+
newTable);
466+
ImmutableCommitMeta.Builder builder = ImmutableCommitMeta.builder();
467+
builder.message(buildCommitMsg(base, metadata, key.toString()));
468+
if (isSnapshotOperation(base, metadata)) {
469+
builder.putProperties("iceberg.operation", snapshot.operation());
470+
}
471+
Branch branch =
472+
getApi()
473+
.commitMultipleOperations()
474+
.operation(Operation.Put.of(key, newTable, expectedContent))
475+
.commitMeta(NessieUtil.catalogOptions(builder, catalogOptions).build())
476+
.branch(expectedHead)
477+
.commit();
478+
LOG.info(
479+
"Committed '{}' against '{}', expected commit-id was '{}'",
480+
key,
481+
branch,
482+
expectedHead.getHash());
483+
updateableReference.updateReference(branch);
484+
}
485+
486+
private boolean isSnapshotOperation(TableMetadata base, TableMetadata metadata) {
487+
Snapshot snapshot = metadata.currentSnapshot();
488+
return snapshot != null
489+
&& (base == null
490+
|| base.currentSnapshot() == null
491+
|| snapshot.snapshotId() != base.currentSnapshot().snapshotId());
492+
}
493+
494+
private String buildCommitMsg(TableMetadata base, TableMetadata metadata, String tableName) {
495+
if (isSnapshotOperation(base, metadata)) {
496+
return String.format(
497+
"Iceberg %s against %s", metadata.currentSnapshot().operation(), tableName);
498+
} else if (base != null && metadata.currentSchemaId() != base.currentSchemaId()) {
499+
return String.format("Iceberg schema change against %s", tableName);
500+
}
501+
return String.format("Iceberg commit against %s", tableName);
502+
}
503+
504+
public String refName() {
505+
return getRef().getName();
506+
}
507+
419508
@Override
420509
public void close() {
421510
if (null != api) {

‎nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java‎

Lines changed: 4 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020

2121
import java.util.Map;
2222
import org.apache.iceberg.BaseMetastoreTableOperations;
23-
import org.apache.iceberg.Snapshot;
2423
import org.apache.iceberg.SnapshotRef;
2524
import org.apache.iceberg.TableMetadata;
2625
import org.apache.iceberg.TableMetadataParser;
@@ -33,13 +32,9 @@
3332
import org.projectnessie.client.http.HttpClientException;
3433
import org.projectnessie.error.NessieConflictException;
3534
import org.projectnessie.error.NessieNotFoundException;
36-
import org.projectnessie.model.Branch;
3735
import org.projectnessie.model.Content;
3836
import org.projectnessie.model.ContentKey;
3937
import org.projectnessie.model.IcebergTable;
40-
import org.projectnessie.model.ImmutableCommitMeta;
41-
import org.projectnessie.model.ImmutableIcebergTable;
42-
import org.projectnessie.model.Operation;
4338
import org.projectnessie.model.Reference;
4439
import org.slf4j.Logger;
4540
import org.slf4j.LoggerFactory;
@@ -148,75 +143,22 @@ protected void doRefresh() {
148143

149144
@Override
150145
protected void doCommit(TableMetadata base, TableMetadata metadata) {
151-
UpdateableReference updateableReference = client.getRef();
152-
153-
updateableReference.checkMutable();
154-
155-
Branch current = updateableReference.getAsBranch();
156-
Branch expectedHead = current;
157-
if (base != null) {
158-
String metadataCommitId = base.property(NESSIE_COMMIT_ID_PROPERTY, expectedHead.getHash());
159-
if (metadataCommitId != null) {
160-
expectedHead = Branch.of(expectedHead.getName(), metadataCommitId);
161-
}
162-
}
163-
164146
String newMetadataLocation =
165147
(base == null) && (metadata.metadataFileLocation() != null)
166148
? metadata.metadataFileLocation()
167149
: writeNewMetadata(metadata, currentVersion() + 1);
168150

151+
String refName = client.refName();
169152
boolean delete = true;
170153
try {
171-
ImmutableIcebergTable.Builder newTableBuilder = ImmutableIcebergTable.builder();
172-
if (table != null) {
173-
newTableBuilder.id(table.getId());
174-
}
175-
Snapshot snapshot = metadata.currentSnapshot();
176-
long snapshotId = snapshot != null ? snapshot.snapshotId() : -1L;
177-
178-
IcebergTable newTable =
179-
newTableBuilder
180-
.snapshotId(snapshotId)
181-
.schemaId(metadata.currentSchemaId())
182-
.specId(metadata.defaultSpecId())
183-
.sortOrderId(metadata.defaultSortOrderId())
184-
.metadataLocation(newMetadataLocation)
185-
.build();
186-
187-
LOG.debug(
188-
"Committing '{}' against '{}', current is '{}': {}",
189-
key,
190-
expectedHead,
191-
current.getHash(),
192-
newTable);
193-
ImmutableCommitMeta.Builder builder = ImmutableCommitMeta.builder();
194-
builder.message(buildCommitMsg(base, metadata));
195-
if (isSnapshotOperation(base, metadata)) {
196-
builder.putProperties("iceberg.operation", snapshot.operation());
197-
}
198-
Branch branch =
199-
client
200-
.getApi()
201-
.commitMultipleOperations()
202-
.operation(Operation.Put.of(key, newTable, table))
203-
.commitMeta(NessieUtil.catalogOptions(builder, catalogOptions).build())
204-
.branch(expectedHead)
205-
.commit();
206-
LOG.info(
207-
"Committed '{}' against '{}', expected commit-id was '{}'",
208-
key,
209-
branch,
210-
expectedHead.getHash());
211-
updateableReference.updateReference(branch);
212-
154+
client.commitTable(base, metadata, newMetadataLocation, table, key);
213155
delete = false;
214156
} catch (NessieConflictException ex) {
215157
throw new CommitFailedException(
216158
ex,
217159
"Cannot commit: Reference hash is out of date. "
218160
+ "Update the reference '%s' and try again",
219-
updateableReference.getName());
161+
refName);
220162
} catch (HttpClientException ex) {
221163
// Intentionally catch all nessie-client-exceptions here and not just the "timeout" variant
222164
// to catch all kinds of network errors (e.g. connection reset). Network code implementation
@@ -226,34 +168,14 @@ protected void doCommit(TableMetadata base, TableMetadata metadata) {
226168
throw new CommitStateUnknownException(ex);
227169
} catch (NessieNotFoundException ex) {
228170
throw new RuntimeException(
229-
String.format(
230-
"Cannot commit: Reference '%s' no longer exists", updateableReference.getName()),
231-
ex);
171+
String.format("Cannot commit: Reference '%s' no longer exists", refName), ex);
232172
} finally {
233173
if (delete) {
234174
io().deleteFile(newMetadataLocation);
235175
}
236176
}
237177
}
238178

239-
private boolean isSnapshotOperation(TableMetadata base, TableMetadata metadata) {
240-
Snapshot snapshot = metadata.currentSnapshot();
241-
return snapshot != null
242-
&& (base == null
243-
|| base.currentSnapshot() == null
244-
|| snapshot.snapshotId() != base.currentSnapshot().snapshotId());
245-
}
246-
247-
private String buildCommitMsg(TableMetadata base, TableMetadata metadata) {
248-
if (isSnapshotOperation(base, metadata)) {
249-
return String.format(
250-
"Iceberg %s against %s", metadata.currentSnapshot().operation(), tableName());
251-
} else if (base != null && metadata.currentSchemaId() != base.currentSchemaId()) {
252-
return String.format("Iceberg schema change against %s", tableName());
253-
}
254-
return String.format("Iceberg commit against %s", tableName());
255-
}
256-
257179
@Override
258180
public FileIO io() {
259181
return fileIO;

0 commit comments

Comments
 (0)