add backupmeta decoder

Signed-off-by: marsishandsome <marsishandsome@gmail.com>

Author: marsishandsome

src/main/java/org/tikv/br/BackupDecoder.java

@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import org.rocksdb.Options;
+import org.rocksdb.ReadOptions;
+import org.tikv.kvproto.BackupOuterClass;
+
+public class BackupDecoder {
+  private final BackupOuterClass.BackupMeta backupMeta;
+
+  public BackupDecoder(BackupOuterClass.BackupMeta backupMeta) {
+    this.backupMeta = backupMeta;
+  }
+
+  private KVDecoder getKVDecoder() {
+    // Currently only v1 is supported.
+    // V2 will be added after https://github.com/tikv/tikv/issues/10938.
+    return RawKVDecoderV1.INSTANCE;
+  }
+
+  public SSTDecoder decodeSST(String sstFilePath) {
+    return decodeSST(sstFilePath, new Options(), new ReadOptions());
+  }
+
+  public SSTDecoder decodeSST(String sstFilePath, Options options, ReadOptions readOptions) {
+    return new SSTDecoder(sstFilePath, getKVDecoder(), options, readOptions);
+  }
+
+  public BackupOuterClass.BackupMeta getBackupMeta() {
+    return backupMeta;
+  }
+}

src/main/java/org/tikv/br/BackupMetaDecoder.java

@@ -0,0 +1,41 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import org.tikv.kvproto.BackupOuterClass;
+
+public class BackupMetaDecoder {
+  private final BackupOuterClass.BackupMeta backupMeta;
+
+  public BackupMetaDecoder(byte[] data) throws InvalidProtocolBufferException {
+    this.backupMeta = BackupOuterClass.BackupMeta.parseFrom(data);
+  }
+
+  public BackupOuterClass.BackupMeta getBackupMeta() {
+    return backupMeta;
+  }
+
+  public static BackupMetaDecoder parse(String backupMetaFilePath) throws IOException {
+    byte[] data = Files.readAllBytes(new File(backupMetaFilePath).toPath());
+    return new BackupMetaDecoder(data);
+  }
+}

src/main/java/org/tikv/br/KVDecoder.java

@@ -0,0 +1,26 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import com.google.protobuf.ByteString;
+
+public interface KVDecoder {
+  ByteString decodeKey(byte[] key);
+
+  ByteString decodeValue(byte[] value);
+}

src/main/java/org/tikv/br/RawKVDecoderV1.java

@@ -0,0 +1,50 @@
+/*
+ *
+ * Copyright 2021 PingCAP, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.tikv.br;
+
+import com.google.protobuf.ByteString;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RawKVDecoderV1 implements KVDecoder {
+  private static final Logger logger = LoggerFactory.getLogger(SSTIterator.class);
+
+  public static final RawKVDecoderV1 INSTANCE = new RawKVDecoderV1();
+
+  private RawKVDecoderV1() {}
+
+  @Override
+  public ByteString decodeKey(byte[] key) {
+    if (key == null || key.length == 0) {
+      logger.warn(
+          "skip Key-Value pair because key == null || key.length == 0, key = "
+              + Arrays.toString(key));
+      return null;
+    } else if (key[0] != 'z') {
+      logger.warn("skip Key-Value pair because key[0] != 'z', key = " + Arrays.toString(key));
+      return null;
+    }
+    return ByteString.copyFrom(key, 1, key.length - 1);
+  }
+
+  @Override
+  public ByteString decodeValue(byte[] value) {
+    return ByteString.copyFrom(value);
+  }
+}

…c/main/java/org/tikv/sst/SSTDecoder.java src/main/java/org/tikv/br/SSTDecoder.javasrc/main/java/org/tikv/sst/SSTDecoder.java renamed to src/main/java/org/tikv/br/SSTDecoder.java src/main/java/org/tikv/sst/SSTDecoder.java renamed to src/main/java/org/tikv/br/SSTDecoder.java

@@ -15,7 +15,7 @@
  *
  */
 
-package org.tikv.sst;
+package org.tikv.br;
 
 import com.google.protobuf.ByteString;
 import java.util.Iterator;
@@ -28,20 +28,24 @@
 
 public class SSTDecoder {
   private final String filePath;
+  private final KVDecoder kvDecoder;
   private final Options options;
   private final ReadOptions readOptions;
 
   private SstFileReader sstFileReader;
   private SstFileReaderIterator iterator;
 
-  public SSTDecoder(String filePath) {
-    this.filePath = filePath;
+  public SSTDecoder(String sstFilePath, KVDecoder kvDecoder) {
+    this.filePath = sstFilePath;
+    this.kvDecoder = kvDecoder;
     this.options = new Options();
     this.readOptions = new ReadOptions();
   }
 
-  public SSTDecoder(String filePath, Options options, ReadOptions readOptions) {
+  public SSTDecoder(
+      String filePath, KVDecoder kvDecoder, Options options, ReadOptions readOptions) {
     this.filePath = filePath;
+    this.kvDecoder = kvDecoder;
     this.options = options;
     this.readOptions = readOptions;
   }
@@ -54,7 +58,7 @@ public synchronized Iterator<Pair<ByteString, ByteString>> getIterator() throws
     sstFileReader = new SstFileReader(new Options());
     sstFileReader.open(filePath);
     iterator = sstFileReader.newIterator(new ReadOptions());
-    return new SSTIterator(iterator);
+    return new SSTIterator(iterator, kvDecoder);
   }
 
   public synchronized void close() {

…/main/java/org/tikv/sst/SSTIterator.java …c/main/java/org/tikv/br/SSTIterator.javasrc/main/java/org/tikv/sst/SSTIterator.java renamed to src/main/java/org/tikv/br/SSTIterator.java src/main/java/org/tikv/sst/SSTIterator.java renamed to src/main/java/org/tikv/br/SSTIterator.java

@@ -15,25 +15,22 @@
  *
  */
 
-package org.tikv.sst;
+package org.tikv.br;
 
 import com.google.protobuf.ByteString;
-import java.util.Arrays;
 import java.util.Iterator;
 import org.rocksdb.SstFileReaderIterator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.tikv.common.util.Pair;
 
 public class SSTIterator implements Iterator<Pair<ByteString, ByteString>> {
-  private static final Logger logger = LoggerFactory.getLogger(SSTIterator.class);
-
   private final SstFileReaderIterator iterator;
+  private final KVDecoder kvDecoder;
 
   private Pair<ByteString, ByteString> nextPair;
 
-  public SSTIterator(SstFileReaderIterator iterator) {
+  public SSTIterator(SstFileReaderIterator iterator, KVDecoder kvDecoder) {
     this.iterator = iterator;
+    this.kvDecoder = kvDecoder;
     this.iterator.seekToFirst();
     this.nextPair = processNext();
   }
@@ -52,8 +49,8 @@ public Pair<ByteString, ByteString> next() {
 
   private Pair<ByteString, ByteString> processNext() {
     if (iterator.isValid()) {
-      ByteString key = decodeKey(iterator.key());
-      ByteString value = decodeValue(iterator.value());
+      ByteString key = kvDecoder.decodeKey(iterator.key());
+      ByteString value = kvDecoder.decodeValue(iterator.value());
       iterator.next();
       if (key != null) {
         return Pair.create(key, value);
@@ -64,21 +61,4 @@ private Pair<ByteString, ByteString> processNext() {
       return null;
     }
   }
-
-  private ByteString decodeKey(byte[] key) {
-    if (key == null || key.length == 0) {
-      logger.warn(
-          "skip Key-Value pair because key == null || key.length == 0, key = "
-              + Arrays.toString(key));
-      return null;
-    } else if (key[0] != 'z') {
-      logger.warn("skip Key-Value pair because key[0] != 'z', key = " + Arrays.toString(key));
-      return null;
-    }
-    return ByteString.copyFrom(key, 1, key.length - 1);
-  }
-
-  private ByteString decodeValue(byte[] value) {
-    return ByteString.copyFrom(value);
-  }
 }

src/test/java/org/tikv/br/BackupDecoderTest.java

@@ -0,0 +1,61 @@
+package org.tikv.br;
+
+import com.google.protobuf.ByteString;
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+import org.junit.Assert;
+import org.junit.Test;
+import org.rocksdb.RocksDBException;
+import org.tikv.common.util.Pair;
+import org.tikv.kvproto.BackupOuterClass;
+
+public class BackupDecoderTest {
+
+  private static final int TOTAL_COUNT = 500;
+  private static final String KEY_PREFIX = "test_";
+  private static final String VALUE =
+      "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+
+  private static final String backupmetaFilePath = "src/test/resources/sst/backupmeta";
+  private static final String sst1FilePath =
+      "src/test/resources/sst/1_2_2_7154800cc311f03afd1532e961b9a878dfbb119b104cf4daad5d0c7c0eacb502_1633919546277_default.sst";
+  private static final String sst2FilePath =
+      "src/test/resources/sst/4_8_2_9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08_1633919546278_default.sst";
+
+  @Test
+  public void rawKVSSTDecoderTest() throws RocksDBException, IOException {
+    BackupMetaDecoder backupMetaDecoder = BackupMetaDecoder.parse(backupmetaFilePath);
+    BackupOuterClass.BackupMeta backupMeta = backupMetaDecoder.getBackupMeta();
+
+    BackupDecoder sstBackup = new BackupDecoder(backupMeta);
+
+    decodeSST(sstBackup, sst1FilePath);
+    decodeSST(sstBackup, sst2FilePath);
+  }
+
+  private void decodeSST(BackupDecoder sstBackup, String sst) throws RocksDBException {
+    String fileName = new File(sst).getName();
+    BackupOuterClass.File backupFile =
+        sstBackup
+            .getBackupMeta()
+            .getFilesList()
+            .stream()
+            .filter(a -> a.getName().equals(fileName))
+            .findFirst()
+            .get();
+    Assert.assertEquals(TOTAL_COUNT, backupFile.getTotalKvs());
+
+    SSTDecoder sstDecoder = sstBackup.decodeSST(sst);
+    Iterator<Pair<ByteString, ByteString>> iterator = sstDecoder.getIterator();
+    int count = 0;
+    while (iterator.hasNext()) {
+      Pair<ByteString, ByteString> pair = iterator.next();
+      Assert.assertEquals(VALUE, pair.second.toStringUtf8());
+      Assert.assertTrue(pair.first.toStringUtf8().startsWith(KEY_PREFIX));
+      count += 1;
+    }
+    sstDecoder.close();
+    Assert.assertEquals(TOTAL_COUNT, count);
+  }
+}