Content-Length: 367672 | pFad | http://github.com/apache/iceberg/pull/5984/commits/7f8ee49ad779b028713146fc49bdfe038d2dc2b3

06 Core, API: Support incremental scanning with branch by hililiwei · Pull Request #5984 · apache/iceberg · GitHub
Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Core, API: Support incremental scanning with branch #5984

Merged
merged 9 commits into from
Aug 23, 2023
Merged

Core, API: Support incremental scanning with branch #5984

Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
68ddf2c
Core, API: Support incremental scanning with branch
hililiwei Oct 14, 2022
9f0a543
address comments
hililiwei Feb 17, 2023
04c76c4
fix ut
hililiwei Apr 26, 2023
9be8122
make default instead
hililiwei Apr 27, 2023
24ff240
rebase and address
hililiwei Apr 27, 2023
349087e
rebase and address
hililiwei Jul 28, 2023
7f8ee49
rebase and address
hililiwei Jul 28, 2023
1c42627
rebase and address
hililiwei Jul 29, 2023
22c1ab5
rebase and address
hililiwei Jul 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rebase and address
  • Loading branch information
@hililiwei
hililiwei committed Jul 28, 2023
commit 7f8ee49ad779b028713146fc49bdfe038d2dc2b3
129 changes: 58 additions & 71 deletions core/src/test/java/org/apache/iceberg/TestBaseIncrementalAppendScan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,15 +56,6 @@ public void fromSnapshotInclusiveWithNonExistingRef() {
Assertions.assertThatThrownBy(() -> newScan().fromSnapshotInclusive("nonExistingRef"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot find ref: nonExistingRef");

table.newFastAppend().appendFile(FILE_A).commit();
Assertions.assertThatThrownBy(
() ->
newScan()
.fromSnapshotInclusive(table.currentSnapshot().snapshotId())
.toSnapshot("nonExistingRef"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot find ref: nonExistingRef");
}

@Test
Expand Down Expand Up @@ -110,65 +101,6 @@ public void fromSnapshotInclusiveWithBranchShouldFail() {
.hasMessage(String.format("Ref %s is not a tag", branchName));
}

@Test
public void fromSnapshotExclusiveWithNonExistingRef() {
Assertions.assertThatThrownBy(() -> newScan().fromSnapshotExclusive("nonExistingRef"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot find ref: nonExistingRef");

table.newFastAppend().appendFile(FILE_A).commit();
Assertions.assertThatThrownBy(
() ->
newScan()
.fromSnapshotExclusive(table.currentSnapshot().snapshotId())
.toSnapshot("nonExistingRef"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot find ref: nonExistingRef");
}

@Test
public void testFromSnapshotExclusiveWithTag() {
table.newFastAppend().appendFile(FILE_A).commit();
long snapshotAId = table.currentSnapshot().snapshotId();

String tagSnapshotAName = "t1";
table.manageSnapshots().createTag(tagSnapshotAName, snapshotAId).commit();

String tagSnapshotBName = "t2";
table.newFastAppend().appendFile(FILE_B).appendFile(FILE_B).commit();
long snapshotBId = table.currentSnapshot().snapshotId();
table.manageSnapshots().createTag(tagSnapshotBName, snapshotBId).commit();
table.newFastAppend().appendFile(FILE_C).appendFile(FILE_C).commit();

/*
files:FILE_A files:FILE_B FILE_B files:FILE_C FILE_C
---- snapshotAId(tag:t1) ---- snapshotMainB(tag:t2) ---- currentSnapshot
*/
IncrementalAppendScan scan = newScan().fromSnapshotExclusive(tagSnapshotAName);
Assertions.assertThat(scan.planFiles()).hasSize(4);

IncrementalAppendScan scanWithToSnapshot =
newScan().fromSnapshotExclusive(tagSnapshotAName).toSnapshot(tagSnapshotBName);
Assertions.assertThat(scanWithToSnapshot.planFiles()).hasSize(2);
}

@Test
public void fromSnapshotExclusiveWithBranchShouldFail() {
table.newFastAppend().appendFile(FILE_A).commit();
long snapshotAId = table.currentSnapshot().snapshotId();

String branchName = "b1";
table.manageSnapshots().createBranch(branchName, snapshotAId).commit();
Assertions.assertThatThrownBy(() -> newScan().fromSnapshotExclusive(branchName))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage(String.format("Ref %s is not a tag", branchName));

Assertions.assertThatThrownBy(
() -> newScan().fromSnapshotExclusive(snapshotAId).toSnapshot(branchName))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage(String.format("Ref %s is not a tag", branchName));
}

@Test
public void testUseBranch() {
table.newFastAppend().appendFile(FILE_A).commit();
Expand Down Expand Up @@ -258,8 +190,8 @@ public void testUseBranchWithInvalidEndSnapshotShouldFail() {

/*

files:FILE_A files:FILE_B FILE_B files:FILE_B FILE_B
---- snapshotA(tag:t1) ---- snapshotMainB(tag:t2) ---- currentSnapshot
files:FILE_A files:FILE_B FILE_B
---- snapshotA(tag:t1) ---- snapshotMainB(tag:t2)
\
\
\files:FILE_C
Expand Down Expand Up @@ -315,6 +247,51 @@ public void testFromSnapshotExclusiveForExpiredParent() {
Assert.assertEquals(1, Iterables.size(scanWithToSnapshot.planFiles()));
}

@Test
public void fromSnapshotExclusiveWithNonExistingRef() {
Assertions.assertThatThrownBy(() -> newScan().fromSnapshotExclusive("nonExistingRef"))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot find ref: nonExistingRef");
}

@Test
public void testFromSnapshotExclusiveWithTag() {
table.newFastAppend().appendFile(FILE_A).commit();
long snapshotAId = table.currentSnapshot().snapshotId();

String tagSnapshotAName = "t1";
table.manageSnapshots().createTag(tagSnapshotAName, snapshotAId).commit();

String tagSnapshotBName = "t2";
table.newFastAppend().appendFile(FILE_B).appendFile(FILE_B).commit();
long snapshotBId = table.currentSnapshot().snapshotId();
table.manageSnapshots().createTag(tagSnapshotBName, snapshotBId).commit();
table.newFastAppend().appendFile(FILE_C).appendFile(FILE_C).commit();

/*
files:FILE_A files:FILE_B FILE_B files:FILE_C FILE_C
---- snapshotAId(tag:t1) ---- snapshotMainB(tag:t2) ---- currentSnapshot
*/
IncrementalAppendScan scan = newScan().fromSnapshotExclusive(tagSnapshotAName);
Assertions.assertThat(scan.planFiles()).hasSize(4);

IncrementalAppendScan scanWithToSnapshot =
newScan().fromSnapshotExclusive(tagSnapshotAName).toSnapshot(tagSnapshotBName);
Assertions.assertThat(scanWithToSnapshot.planFiles()).hasSize(2);
}

@Test
public void fromSnapshotExclusiveWithBranchShouldFail() {
table.newFastAppend().appendFile(FILE_A).commit();
long snapshotAId = table.currentSnapshot().snapshotId();

String branchName = "b1";
table.manageSnapshots().createBranch(branchName, snapshotAId).commit();
Assertions.assertThatThrownBy(() -> newScan().fromSnapshotExclusive(branchName))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage(String.format("Ref %s is not a tag", branchName));
}

@Test
public void testToSnapshot() {
table.newFastAppend().appendFile(FILE_A).commit();
Expand All @@ -329,7 +306,7 @@ public void testToSnapshot() {
}

@Test
public void testToSnapshotWithRef() {
public void testToSnapshotWithTag() {
table.newFastAppend().appendFile(FILE_A).commit();
long snapshotAId = table.currentSnapshot().snapshotId();
hililiwei marked this conversation as resolved.
Show resolved Hide resolved
table.newFastAppend().appendFile(FILE_B).commit();
Expand All @@ -340,12 +317,22 @@ public void testToSnapshotWithRef() {

String tagSnapshotMainBName = "t1";
table.manageSnapshots().createTag(tagSnapshotMainBName, snapshotBId).commit();
table.newFastAppend().appendFile(FILE_B).appendFile(FILE_B).commit();

String tagSnapshotBranchBName = "t2";
table.newFastAppend().appendFile(FILE_C).toBranch(branchName).commit();
long snapshotBranchBId = table.snapshot(branchName).snapshotId();
table.manageSnapshots().createTag(tagSnapshotBranchBName, snapshotBranchBId).commit();

/*

files:FILE_A files:FILE_B files:FILE_B FILE_B
----snapshotA ------ snapshotMainB(tag:t1) -------- currentSnapshot
\
\
\files:FILE_C
snapshotBranchB(branch:b1, tag:t2)
*/
IncrementalAppendScan scan = newScan().toSnapshot(tagSnapshotMainBName);
Assertions.assertThat(scan.planFiles()).hasSize(2);

Expand Down








ApplySandwichStrip

pFad - (p)hone/(F)rame/(a)nonymizer/(d)eclutterfier!      Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

Fetched URL: http://github.com/apache/iceberg/pull/5984/commits/7f8ee49ad779b028713146fc49bdfe038d2dc2b3

Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy