Skip to content

Commit 1145317

Browse files
shizy818JackieTien97
shizy818
authored and
JackieTien97
committed
fix: clone & delete issues (#15153)
* fix: delete column bug for aligned timeseries * fix: clone data types list when create new AlignedTVList * fix: bitmap is null when delete column * init bitmaps when it is null * fix: error during rebase code * set memchunk datatypes when handover (cherry picked from commit 7bfcb49)
1 parent 6d65b2e commit 1145317

4 files changed

Lines changed: 164 additions & 121 deletions

File tree

‎integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBDeleteAlignedTimeseriesIT.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,4 +242,46 @@ public void deleteTimeSeriesMultiIntervalTest() {
242242
fail(e.getMessage());
243243
}
244244
}
245+
246+
@Test
247+
public void deleteTimeseriesAndCreateSameTypeTest2() throws Exception {
248+
String[] retArray = new String[] {"1,4.0,", "2,8.0,"};
249+
int cnt = 0;
250+
try (Connection connection = EnvFactory.getEnv().getConnection();
251+
Statement statement = connection.createStatement()) {
252+
statement.execute(
253+
"create aligned timeseries root.turbine1.d1(s1 FLOAT encoding=PLAIN compression=SNAPPY, "
254+
+ "s2 INT64 encoding=PLAIN compression=SNAPPY, s4 DOUBLE encoding=PLAIN compression=SNAPPY)");
255+
statement.execute("INSERT INTO root.turbine1.d1(timestamp,s1,s2,s4) ALIGNED VALUES(1,1,2,4)");
256+
257+
try (ResultSet resultSet = statement.executeQuery("SELECT s4 FROM root.turbine1.d1")) {
258+
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
259+
while (resultSet.next()) {
260+
StringBuilder builder = new StringBuilder();
261+
for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
262+
builder.append(resultSet.getString(i)).append(",");
263+
}
264+
Assert.assertEquals(retArray[cnt], builder.toString());
265+
cnt++;
266+
}
267+
}
268+
// delete series in the middle
269+
statement.execute("DELETE timeseries root.turbine1.d1.s4");
270+
statement.execute(
271+
"INSERT INTO root.turbine1.d1(timestamp,s3,s4) ALIGNED VALUES(2,false,8.0)");
272+
statement.execute("FLUSH");
273+
274+
try (ResultSet resultSet = statement.executeQuery("SELECT s4 FROM root.turbine1.d1")) {
275+
ResultSetMetaData resultSetMetaData = resultSet.getMetaData();
276+
while (resultSet.next()) {
277+
StringBuilder builder = new StringBuilder();
278+
for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) {
279+
builder.append(resultSet.getString(i)).append(",");
280+
}
281+
Assert.assertEquals(retArray[cnt], builder.toString());
282+
cnt++;
283+
}
284+
}
285+
}
286+
}
245287
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java‎

Lines changed: 104 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
public class AlignedWritableMemChunk extends AbstractWritableMemChunk {
6262

6363
private final Map<String, Integer> measurementIndexMap;
64-
private final List<TSDataType> dataTypes;
64+
private List<TSDataType> dataTypes;
6565
private final List<IMeasurementSchema> schemaList;
6666
private AlignedTVList list;
6767
private List<AlignedTVList> sortedList;
@@ -197,7 +197,8 @@ protected void handoverAlignedTvList() {
197197
list.sort();
198198
}
199199
sortedList.add(list);
200-
this.list = AlignedTVList.newAlignedList(dataTypes);
200+
this.list = AlignedTVList.newAlignedList(new ArrayList<>(dataTypes));
201+
this.dataTypes = list.getTsDataTypes();
201202
}
202203

203204
@Override
@@ -230,6 +231,107 @@ public void writeAlignedTablet(
230231
}
231232
}
232233

234+
/**
235+
* Check metadata of columns and return array that mapping existed metadata to index of data
236+
* column.
237+
*
238+
* @param schemaListInInsertPlan Contains all existed schema in InsertPlan. If some timeseries
239+
* have been deleted, there will be null in its slot.
240+
* @return columnIndexArray: schemaList[i] is schema of columns[columnIndexArray[i]]
241+
*/
242+
private Pair<Object[], BitMap[]> checkAndReorderColumnValuesInInsertPlan(
243+
List<IMeasurementSchema> schemaListInInsertPlan, Object[] columnValues, BitMap[] bitMaps) {
244+
Object[] reorderedColumnValues = new Object[schemaList.size()];
245+
BitMap[] reorderedBitMaps = bitMaps == null ? null : new BitMap[schemaList.size()];
246+
for (int i = 0; i < schemaListInInsertPlan.size(); i++) {
247+
IMeasurementSchema measurementSchema = schemaListInInsertPlan.get(i);
248+
if (measurementSchema != null) {
249+
Integer index = this.measurementIndexMap.get(measurementSchema.getMeasurementName());
250+
// Index is null means this measurement was not in this AlignedTVList before.
251+
// We need to extend a new column in AlignedMemChunk and AlignedTVList.
252+
// And the reorderedColumnValues should extend one more column for the new measurement
253+
if (index == null) {
254+
index = this.list.getTsDataTypes().size();
255+
this.measurementIndexMap.put(schemaListInInsertPlan.get(i).getMeasurementName(), index);
256+
this.schemaList.add(schemaListInInsertPlan.get(i));
257+
this.list.extendColumn(schemaListInInsertPlan.get(i).getType());
258+
reorderedColumnValues =
259+
Arrays.copyOf(reorderedColumnValues, reorderedColumnValues.length + 1);
260+
if (reorderedBitMaps != null) {
261+
reorderedBitMaps = Arrays.copyOf(reorderedBitMaps, reorderedBitMaps.length + 1);
262+
}
263+
}
264+
reorderedColumnValues[index] = columnValues[i];
265+
if (bitMaps != null) {
266+
reorderedBitMaps[index] = bitMaps[i];
267+
}
268+
}
269+
}
270+
return new Pair<>(reorderedColumnValues, reorderedBitMaps);
271+
}
272+
273+
private void filterDeletedTimeStamp(
274+
AlignedTVList alignedTVList,
275+
List<List<TimeRange>> valueColumnsDeletionList,
276+
boolean ignoreAllNullRows,
277+
Map<Long, BitMap> timestampWithBitmap) {
278+
BitMap allValueColDeletedMap = alignedTVList.getAllValueColDeletedMap();
279+
280+
int rowCount = alignedTVList.rowCount();
281+
List<int[]> valueColumnDeleteCursor = new ArrayList<>();
282+
if (valueColumnsDeletionList != null) {
283+
valueColumnsDeletionList.forEach(x -> valueColumnDeleteCursor.add(new int[] {0}));
284+
}
285+
286+
for (int row = 0; row < rowCount; row++) {
287+
// the row is deleted
288+
if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(row)) {
289+
continue;
290+
}
291+
long timestamp = alignedTVList.getTime(row);
292+
293+
BitMap bitMap = new BitMap(schemaList.size());
294+
for (int column = 0; column < schemaList.size(); column++) {
295+
if (alignedTVList.isNullValue(alignedTVList.getValueIndex(row), column)) {
296+
bitMap.mark(column);
297+
}
298+
299+
// skip deleted row
300+
if (valueColumnsDeletionList != null
301+
&& !valueColumnsDeletionList.isEmpty()
302+
&& isPointDeleted(
303+
timestamp,
304+
valueColumnsDeletionList.get(column),
305+
valueColumnDeleteCursor.get(column))) {
306+
bitMap.mark(column);
307+
}
308+
309+
// skip all-null row
310+
if (ignoreAllNullRows && bitMap.isAllMarked()) {
311+
continue;
312+
}
313+
timestampWithBitmap.put(timestamp, bitMap);
314+
}
315+
}
316+
}
317+
318+
public long[] getFilteredTimestamp(
319+
List<List<TimeRange>> deletionList, List<BitMap> bitMaps, boolean ignoreAllNullRows) {
320+
Map<Long, BitMap> timestampWithBitmap = new TreeMap<>();
321+
322+
filterDeletedTimeStamp(list, deletionList, ignoreAllNullRows, timestampWithBitmap);
323+
for (AlignedTVList alignedTVList : sortedList) {
324+
filterDeletedTimeStamp(alignedTVList, deletionList, ignoreAllNullRows, timestampWithBitmap);
325+
}
326+
327+
List<Long> filteredTimestamps = new ArrayList<>();
328+
for (Map.Entry<Long, BitMap> entry : timestampWithBitmap.entrySet()) {
329+
filteredTimestamps.add(entry.getKey());
330+
bitMaps.add(entry.getValue());
331+
}
332+
return filteredTimestamps.stream().mapToLong(Long::valueOf).toArray();
333+
}
334+
233335
@Override
234336
public AlignedTVList getWorkingTVList() {
235337
return list;
@@ -785,112 +887,4 @@ public List<Integer> buildColumnIndexList(List<IMeasurementSchema> schemaList) {
785887
}
786888
return columnIndexList;
787889
}
788-
789-
/**
790-
* Check metadata of columns and return array that mapping existed metadata to index of data
791-
* column.
792-
*
793-
* @param schemaListInInsertPlan Contains all existed schema in InsertPlan. If some timeseries
794-
* have been deleted, there will be null in its slot.
795-
* @return columnIndexArray: schemaList[i] is schema of columns[columnIndexArray[i]]
796-
*/
797-
private Pair<Object[], BitMap[]> checkAndReorderColumnValuesInInsertPlan(
798-
List<IMeasurementSchema> schemaListInInsertPlan, Object[] columnValues, BitMap[] bitMaps) {
799-
Object[] reorderedColumnValues = new Object[schemaList.size()];
800-
BitMap[] reorderedBitMaps = bitMaps == null ? null : new BitMap[schemaList.size()];
801-
for (int i = 0; i < schemaListInInsertPlan.size(); i++) {
802-
IMeasurementSchema measurementSchema = schemaListInInsertPlan.get(i);
803-
if (measurementSchema != null) {
804-
Integer index = this.measurementIndexMap.get(measurementSchema.getMeasurementName());
805-
// Index is null means this measurement was not in this AlignedTVList before.
806-
// We need to extend a new column in AlignedMemChunk and AlignedTVList.
807-
// And the reorderedColumnValues should extend one more column for the new measurement
808-
if (index == null) {
809-
index =
810-
measurementIndexMap.isEmpty()
811-
? 0
812-
: measurementIndexMap.values().stream()
813-
.mapToInt(Integer::intValue)
814-
.max()
815-
.getAsInt()
816-
+ 1;
817-
this.measurementIndexMap.put(schemaListInInsertPlan.get(i).getMeasurementName(), index);
818-
this.schemaList.add(schemaListInInsertPlan.get(i));
819-
this.list.extendColumn(schemaListInInsertPlan.get(i).getType());
820-
reorderedColumnValues =
821-
Arrays.copyOf(reorderedColumnValues, reorderedColumnValues.length + 1);
822-
if (reorderedBitMaps != null) {
823-
reorderedBitMaps = Arrays.copyOf(reorderedBitMaps, reorderedBitMaps.length + 1);
824-
}
825-
}
826-
reorderedColumnValues[index] = columnValues[i];
827-
if (bitMaps != null) {
828-
reorderedBitMaps[index] = bitMaps[i];
829-
}
830-
}
831-
}
832-
return new Pair<>(reorderedColumnValues, reorderedBitMaps);
833-
}
834-
835-
private void filterDeletedTimeStamp(
836-
AlignedTVList alignedTVList,
837-
List<List<TimeRange>> valueColumnsDeletionList,
838-
boolean ignoreAllNullRows,
839-
Map<Long, BitMap> timestampWithBitmap) {
840-
BitMap allValueColDeletedMap = alignedTVList.getAllValueColDeletedMap();
841-
842-
int rowCount = alignedTVList.rowCount();
843-
List<int[]> valueColumnDeleteCursor = new ArrayList<>();
844-
if (valueColumnsDeletionList != null) {
845-
valueColumnsDeletionList.forEach(x -> valueColumnDeleteCursor.add(new int[] {0}));
846-
}
847-
848-
for (int row = 0; row < rowCount; row++) {
849-
// the row is deleted
850-
if (allValueColDeletedMap != null && allValueColDeletedMap.isMarked(row)) {
851-
continue;
852-
}
853-
long timestamp = alignedTVList.getTime(row);
854-
855-
BitMap bitMap = new BitMap(schemaList.size());
856-
for (int column = 0; column < schemaList.size(); column++) {
857-
if (alignedTVList.isNullValue(alignedTVList.getValueIndex(row), column)) {
858-
bitMap.mark(column);
859-
}
860-
861-
// skip deleted row
862-
if (valueColumnsDeletionList != null
863-
&& !valueColumnsDeletionList.isEmpty()
864-
&& isPointDeleted(
865-
timestamp,
866-
valueColumnsDeletionList.get(column),
867-
valueColumnDeleteCursor.get(column))) {
868-
bitMap.mark(column);
869-
}
870-
871-
// skip all-null row
872-
if (ignoreAllNullRows && bitMap.isAllMarked()) {
873-
continue;
874-
}
875-
timestampWithBitmap.put(timestamp, bitMap);
876-
}
877-
}
878-
}
879-
880-
public long[] getFilteredTimestamp(
881-
List<List<TimeRange>> deletionList, List<BitMap> bitMaps, boolean ignoreAllNullRows) {
882-
Map<Long, BitMap> timestampWithBitmap = new TreeMap<>();
883-
884-
filterDeletedTimeStamp(list, deletionList, ignoreAllNullRows, timestampWithBitmap);
885-
for (AlignedTVList alignedTVList : sortedList) {
886-
filterDeletedTimeStamp(alignedTVList, deletionList, ignoreAllNullRows, timestampWithBitmap);
887-
}
888-
889-
List<Long> filteredTimestamps = new ArrayList<>();
890-
for (Map.Entry<Long, BitMap> entry : timestampWithBitmap.entrySet()) {
891-
filteredTimestamps.add(entry.getKey());
892-
bitMaps.add(entry.getValue());
893-
}
894-
return filteredTimestamps.stream().mapToLong(Long::valueOf).toArray();
895-
}
896890
}

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ public TVList getTvListByColumnIndex(
131131
}
132132
}
133133
}
134-
AlignedTVList alignedTvList = AlignedTVList.newAlignedList(dataTypeList);
134+
AlignedTVList alignedTvList = AlignedTVList.newAlignedList(new ArrayList<>(dataTypeList));
135135
alignedTvList.timestamps = this.timestamps;
136136
alignedTvList.indices = this.indices;
137137
alignedTvList.values = values;
@@ -147,7 +147,7 @@ public TVList getTvListByColumnIndex(
147147

148148
@Override
149149
public synchronized AlignedTVList clone() {
150-
AlignedTVList cloneList = AlignedTVList.newAlignedList(dataTypes);
150+
AlignedTVList cloneList = AlignedTVList.newAlignedList(new ArrayList<>(dataTypes));
151151
cloneAs(cloneList);
152152
cloneList.timeDeletedCnt = this.timeDeletedCnt;
153153
System.arraycopy(
@@ -346,10 +346,11 @@ private TsPrimitiveType getAlignedValueByValueIndex(
346346

347347
public void extendColumn(TSDataType dataType) {
348348
if (bitMaps == null) {
349-
bitMaps = new ArrayList<>(values.size());
349+
List<List<BitMap>> localBitMaps = new ArrayList<>(values.size());
350350
for (int i = 0; i < values.size(); i++) {
351-
bitMaps.add(null);
351+
localBitMaps.add(null);
352352
}
353+
bitMaps = localBitMaps;
353354
}
354355
List<Object> columnValue = new ArrayList<>();
355356
List<BitMap> columnBitMaps = new ArrayList<>();
@@ -611,10 +612,11 @@ public Pair<Integer, Boolean> delete(long lowerBound, long upperBound, int colum
611612

612613
public void deleteColumn(int columnIndex) {
613614
if (bitMaps == null) {
614-
bitMaps = new ArrayList<>(dataTypes.size());
615+
List<List<BitMap>> localBitMaps = new ArrayList<>(dataTypes.size());
615616
for (int j = 0; j < dataTypes.size(); j++) {
616-
bitMaps.add(null);
617+
localBitMaps.add(null);
617618
}
619+
bitMaps = localBitMaps;
618620
}
619621
if (bitMaps.get(columnIndex) == null) {
620622
List<BitMap> columnBitMaps = new ArrayList<>();
@@ -624,6 +626,9 @@ public void deleteColumn(int columnIndex) {
624626
bitMaps.set(columnIndex, columnBitMaps);
625627
}
626628
for (int i = 0; i < bitMaps.get(columnIndex).size(); i++) {
629+
if (bitMaps.get(columnIndex).get(i) == null) {
630+
bitMaps.get(columnIndex).set(i, new BitMap(ARRAY_SIZE));
631+
}
627632
bitMaps.get(columnIndex).get(i).markAll();
628633
}
629634
}
@@ -867,10 +872,11 @@ private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex
867872
private void markNullValue(int columnIndex, int arrayIndex, int elementIndex) {
868873
// init BitMaps if doesn't have
869874
if (bitMaps == null) {
870-
bitMaps = new ArrayList<>(dataTypes.size());
875+
List<List<BitMap>> localBitMaps = new ArrayList<>(dataTypes.size());
871876
for (int i = 0; i < dataTypes.size(); i++) {
872-
bitMaps.add(null);
877+
localBitMaps.add(null);
873878
}
879+
bitMaps = localBitMaps;
874880
}
875881

876882
// if the bitmap in columnIndex is null, init the bitmap of this column from the beginning
@@ -1398,7 +1404,7 @@ public static AlignedTVList deserialize(DataInputStream stream) throws IOExcepti
13981404
bitMaps[columnIndex] = bitMap;
13991405
}
14001406

1401-
AlignedTVList tvList = AlignedTVList.newAlignedList(dataTypes);
1407+
AlignedTVList tvList = AlignedTVList.newAlignedList(new ArrayList<>(dataTypes));
14021408
tvList.putAlignedValues(times, values, bitMaps, 0, rowCount, null);
14031409

14041410
boolean hasTimeColDeletedMap = stream.read() == 1;

‎iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,10 +245,11 @@ public int getValueIndex(int index) {
245245
protected void markNullValue(int arrayIndex, int elementIndex) {
246246
// init bitMap if doesn't have
247247
if (bitMap == null) {
248-
bitMap = new ArrayList<>();
248+
List<BitMap> localBitMap = new ArrayList<>();
249249
for (int i = 0; i < timestamps.size(); i++) {
250-
bitMap.add(new BitMap(ARRAY_SIZE));
250+
localBitMap.add(new BitMap(ARRAY_SIZE));
251251
}
252+
bitMap = localBitMap;
252253
}
253254
// if the bitmap in arrayIndex is null, init the bitmap
254255
if (bitMap.get(arrayIndex) == null) {

0 commit comments

Comments
 (0)