Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import org.apache.flink.connector.lance.converter.LanceTypeConverter;
import org.apache.flink.connector.lance.converter.RowDataConverter;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.LogicalTypeRoot;
import org.apache.flink.table.types.logical.RowType;

import org.apache.arrow.memory.BufferAllocator;
Expand Down Expand Up @@ -60,9 +62,12 @@
* 都能用它建表(failover 后新 Writer 实例构造出的 committable 同样可建表)。
* </ul>
*
* <p>孤儿文件:{@link Fragment#write()} 落盘的 {@code .lance} 文件在 TM crash 的
* pre-commit 窗口内不被 commit 时会成为孤儿;交由 Lance 外部 compaction/cleanup
* 清理。此行为与根 src/1.x 版 {@code LanceSink} 完全一致。
* <p><b>孤儿文件防护(#87)</b>:本类不再在 {@link #write(RowData, Context)} 内因
* {@code write.batch-size} 满而物化 {@code .lance} 文件;行先缓冲在内存,物化统一延迟到
* {@link #flush(boolean)} / {@link #prepareCommit()}。这样 TM 在 {@code write()} 与
* {@code prepareCommit()} 之间 crash 时,磁盘上还没有任何文件可成为孤儿,writer 保持无状态。
* 仅当估算的内存缓冲超过 {@code write.memory-cap-bytes} 时才会提前物化(OOM 安全阀,
* 会重新打开一个窄的 orphan 窗口;默认 -1 无上限,永不提前物化)。
*/
public class LanceSinkWriter implements CommittingSinkWriter<RowData, LanceCommittable> {

Expand All @@ -79,9 +84,15 @@ public class LanceSinkWriter implements CommittingSinkWriter<RowData, LanceCommi
/** 当前 checkpoint 窗口内已 flush 产生、待 commit 的 fragments。 */
private final List<FragmentMetadata> pendingFragments = new ArrayList<>();

/** 行级 buffer:满 batchSize 或 flush 时触发 Lance 写。 */
/** 行级内存 buffer:仅在 flush/prepareCommit 或 memory-cap 触发时物化。 */
private final List<RowData> rowBuffer;

/** 每行的估算字节数(基于 rowType 字段类型的一次性估算,用于 memory-cap 触发)。 */
private final long rowBytesEstimate;

/** 当前 rowBuffer 的累计估算字节数。 */
private long estimatedBufferedBytes;

private long totalRowsWritten;

public LanceSinkWriter(LanceOptions options, RowType rowType, int subtaskId) {
Expand All @@ -98,6 +109,7 @@ public LanceSinkWriter(LanceOptions options, RowType rowType, int subtaskId) {
this.converter = new RowDataConverter(rowType);
this.arrowSchema = LanceTypeConverter.toArrowSchema(rowType);
this.rowBuffer = new ArrayList<>(options.getWriteBatchSize());
this.rowBytesEstimate = estimateRowBytes(rowType);

// OVERWRITE 预清理:必须发生在 Fragment.write() 之前。只允许 subtask 0 执行,
// 其它 subtask 可能稍晚启动,等它们开始 write 时 drop 已完成。这里走 lance 侧
Expand Down Expand Up @@ -125,7 +137,13 @@ public LanceSinkWriter(LanceOptions options, RowType rowType, int subtaskId) {
@Override
public void write(RowData element, Context context) throws IOException, InterruptedException {
rowBuffer.add(element);
if (rowBuffer.size() >= options.getWriteBatchSize()) {
estimatedBufferedBytes += rowBytesEstimate;

// #87: 不再因 write.batch-size 满而物化;仅当内存缓冲超过 write.memory-cap-bytes
// 时提前物化(OOM 安全阀)。默认 -1(无上限)→ write() 阶段永不物化,彻底消除
// write() 与 prepareCommit() 之间的 orphan 文件窗口。
long memoryCap = options.getWriteMemoryCapBytes();
if (memoryCap >= 0 && estimatedBufferedBytes >= memoryCap) {
writeBufferAsFragment();
}
}
Expand Down Expand Up @@ -197,6 +215,52 @@ private void writeBufferAsFragment() throws IOException {
"Failed to write Lance fragment for path " + options.getPath(), e);
} finally {
rowBuffer.clear();
estimatedBufferedBytes = 0;
}
}

/**
* 估算单行的逻辑字节数(用于 memory-cap 触发判断,非精确值)。基于 rowType 的字段
* 类型给每个字段一个固定估算,可变长类型(字符串/二进制/复合类型等)取保守默认值。
*/
private static long estimateRowBytes(RowType rowType) {
long bytes = 16; // RowData 对象本身 + 内部数组的固定 overhead
for (LogicalType type : rowType.getChildren()) {
bytes += estimateTypeBytes(type.getTypeRoot());
}
return Math.max(1, bytes);
}

private static long estimateTypeBytes(LogicalTypeRoot root) {
switch (root) {
case TINYINT:
case BOOLEAN:
return 1;
case SMALLINT:
return 2;
case INTEGER:
case FLOAT:
case DATE:
case TIME_WITHOUT_TIME_ZONE:
return 4;
case BIGINT:
case DOUBLE:
case TIMESTAMP_WITHOUT_TIME_ZONE:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
case TIMESTAMP_WITH_TIME_ZONE:
case INTERVAL_YEAR_MONTH:
case INTERVAL_DAY_TIME:
return 8;
case DECIMAL:
return 16;
case CHAR:
case VARCHAR:
case BINARY:
case VARBINARY:
return 32; // 可变长类型取保守估算
default:
// ARRAY / MAP / ROW / MULTISET / RAW / DISTINCT / STRUCTURED_TYPE / NULL 等
return 64; // 复合类型取保守估算
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.flink.connector.lance.config;

import org.apache.flink.configuration.Configuration;

import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

/**
* #87 regression tests for {@code write.memory-cap-bytes}.
*
* <p>The option gates the writer's OOM safety valve: -1 means unbounded (write() never
* materializes, eliminating the orphan-file window); a non-negative value bounds the
* in-memory row buffer, materializing early when the estimate is exceeded.
*/
class LanceOptionsWriteMemoryCapTest {

@Test
@DisplayName("write.memory-cap-bytes defaults to -1 (unbounded, write() never materializes)")
void defaultsToUnbounded() {
LanceOptions options =
LanceOptions.builder()
.path("/tmp/x")
.build();

assertThat(options.getWriteMemoryCapBytes()).isEqualTo(-1L);
}

@Test
@DisplayName("builder writeMemoryCapBytes round-trips")
void builderRoundTrip() {
LanceOptions options =
LanceOptions.builder()
.path("/tmp/x")
.writeMemoryCapBytes(64 * 1024 * 1024L)
.build();

assertThat(options.getWriteMemoryCapBytes()).isEqualTo(64 * 1024 * 1024L);
}

@Test
@DisplayName("fromConfiguration reads write.memory-cap-bytes")
void fromConfigurationRoundTrip() {
Configuration config = new Configuration();
config.set(LanceOptions.WRITE_MEMORY_CAP_BYTES, 128 * 1024 * 1024L);

LanceOptions options = LanceOptions.fromConfiguration(config);

assertThat(options.getWriteMemoryCapBytes()).isEqualTo(128 * 1024 * 1024L);
}

@Test
@DisplayName("write.memory-cap-bytes below -1 is rejected")
void rejectsBelowMinusOne() {
LanceOptions.Builder builder =
LanceOptions.builder().path("/tmp/x").writeMemoryCapBytes(-2L);

assertThatThrownBy(builder::build)
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("write.memory-cap-bytes");
}

@Test
@DisplayName("write.memory-cap-bytes = 0 is a valid bounded cap")
void zeroIsValidBoundedCap() {
LanceOptions options =
LanceOptions.builder()
.path("/tmp/x")
.writeMemoryCapBytes(0L)
.build();

assertThat(options.getWriteMemoryCapBytes()).isEqualTo(0L);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.flink.connector.lance.sink;

import org.apache.flink.connector.lance.config.LanceOptions;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.StringData;
import org.apache.flink.table.types.logical.BigIntType;
import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.table.types.logical.VarCharType;

import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.stream.Stream;

import static org.assertj.core.api.Assertions.assertThat;

/**
* #87 regression tests for {@link LanceSinkWriter} orphan-file prevention.
*
* <p>The core behavioural change: {@code write()} must <b>not</b> materialize {@code .lance}
* files on disk (previously it did once {@code write.batch-size} was reached). Materialization
* is deferred to {@code flush()} / {@code prepareCommit()}, so a TM crash between
* {@code write()} and {@code prepareCommit()} leaves no orphan files behind.
*
* <p>These tests exercise the real Lance JNI (fragment materialization), which is the exact
* side effect the orphan leak depends on. They are the direct, deterministic equivalent of the
* issue's "kill TM between write() and prepareCommit()" MiniCluster scenario: if {@code write()}
* produces no {@code .lance} files, a crash at that point cannot orphan anything.
*/
class LanceSinkWriterOrphanFileTest {

@TempDir
Path tempDir;

private static final RowType ROW_TYPE =
new RowType(
Arrays.asList(
new RowType.RowField("id", new BigIntType(false)),
new RowType.RowField("content", new VarCharType(true, VarCharType.MAX_LENGTH))));

private LanceOptions options(String path) {
return LanceOptions.builder()
.path(path)
.writeBatchSize(1024)
.writeMode(LanceOptions.WriteMode.APPEND)
.build();
}

private GenericRowData row(long id, String content) {
GenericRowData row = new GenericRowData(2);
row.setField(0, id);
row.setField(1, StringData.fromString(content));
return row;
}

private static long countLanceFiles(Path dir) throws IOException {
if (!Files.exists(dir)) {
return 0;
}
try (Stream<Path> paths = Files.walk(dir)) {
return paths.filter(p -> p.toString().endsWith(".lance")).count();
}
}

@Test
@DisplayName("write() below memory-cap does not materialize any .lance file")
void writeDoesNotMaterialize() throws Exception {
Path datasetPath = tempDir.resolve("no_materialize");
// default write.memory-cap-bytes = -1 (unbounded) → write() never materializes.
try (LanceSinkWriter writer = new LanceSinkWriter(options(datasetPath.toString()), ROW_TYPE, 0)) {
for (int i = 0; i < 500; i++) {
writer.write(row(i, "content-" + i), null);
}

// Core assertion: no .lance file exists yet even though we wrote 500 rows
// (well above the old 1024-row batchSize trigger is not hit, but the old
// behaviour materialized at batchSize; the new behaviour materializes never
// during write).
assertThat(countLanceFiles(datasetPath)).isZero();
}
}

@Test
@DisplayName("flush() materializes the buffered rows")
void flushMaterializes() throws Exception {
Path datasetPath = tempDir.resolve("flush_materialize");
try (LanceSinkWriter writer = new LanceSinkWriter(options(datasetPath.toString()), ROW_TYPE, 0)) {
for (int i = 0; i < 100; i++) {
writer.write(row(i, "content-" + i), null);
}
assertThat(countLanceFiles(datasetPath)).isZero();

writer.flush(false);

assertThat(countLanceFiles(datasetPath)).isPositive();
}
}

@Test
@DisplayName("prepareCommit() materializes the buffered rows")
void prepareCommitMaterializes() throws Exception {
Path datasetPath = tempDir.resolve("prepare_commit_materialize");
try (LanceSinkWriter writer = new LanceSinkWriter(options(datasetPath.toString()), ROW_TYPE, 0)) {
for (int i = 0; i < 100; i++) {
writer.write(row(i, "content-" + i), null);
}
assertThat(countLanceFiles(datasetPath)).isZero();

writer.prepareCommit();

assertThat(countLanceFiles(datasetPath)).isPositive();
}
}

@Test
@DisplayName("write.memory-cap-bytes triggers early materialization as an OOM safety valve")
void memoryCapTriggersEarlyMaterialization() throws Exception {
Path datasetPath = tempDir.resolve("memory_cap_materialize");
// per-row estimate = 16 overhead + 8 (BIGINT) + 32 (VARCHAR) = 56; a tiny cap forces
// materialization on the very first write.
LanceOptions capped =
LanceOptions.builder()
.path(datasetPath.toString())
.writeMode(LanceOptions.WriteMode.APPEND)
.writeMemoryCapBytes(1L)
.build();

try (LanceSinkWriter writer = new LanceSinkWriter(capped, ROW_TYPE, 0)) {
writer.write(row(1, "x"), null);

// The OOM safety valve has already materialized on write().
assertThat(countLanceFiles(datasetPath)).isPositive();
}
}
}
Loading
Loading