ARROW-10832: [Rust] [Arrow] generate src/ipc/gen/* with latest snapshot flatc.

I managed to regenerate the code by snapshot flatc, because flatc 1.12.0 does not work.
Both `cargo test` and `cargo doc` passed.

Closes #8859 from mqy/master

Authored-by: mqy <meng.qingyou@gmail.com>
Signed-off-by: Neville Dipale <nevilledips@gmail.com>

Author: mqy

rust/.gitignore

@@ -2,3 +2,4 @@ Cargo.lock
 target
 rusty-tags.vi
 .history
+.flatbuffers/

rust/arrow/README.md

@@ -68,7 +68,10 @@ cargo run --example read_csv
 
 ## IPC
 
-The IPC flatbuffer code was generated by running this command from the root of the project, using flatc version 1.10.0:
+The expected flatc version is 1.12.0+, built from [flatbuffers](https://github.com/google/flatbuffers)
+master at fixed commit ID, by regen.sh.
+
+The IPC flatbuffer code was generated by running this command from the root of the project:
 
 ```bash
 ./regen.sh

rust/arrow/regen.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -e
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -21,8 +21,43 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # Change to the toplevel Rust directory
 pushd $DIR/../../
 
+# As of 2020-12-06, the snapshot flatc version is not changed since "1.12.0",
+# so let's build flatc from source.
+
+FB_URL="https://github.com/google/flatbuffers"
+FB_COMMIT="2046bffa40400904c926c2a5bedab67a8d6b7e08"
+FB_DIR="rust/arrow/.flatbuffers"
+FLATC="$FB_DIR/bazel-bin/flatc"
+
+if [ ! -e "$FLATC" ]; then
+    echo "$FLATC: not found, let's build it ..."
+
+    if [ -z $(which bazel) ]; then
+        echo "bazel is required to build flatc"
+        exit 1
+    fi
+
+    echo "Bazel version: $(bazel version | head -1 | awk -F':' '{print $2}')"
+
+    if [ ! -e $FB_DIR ]; then
+        echo "git clone $FB_URL ..."
+        git clone -b master --no-tag --depth 1 $FB_URL $FB_DIR
+    else
+        echo "git pull $FB_URL ..."
+        git -C $FB_DIR pull
+    fi
+
+    echo "hard reset to $FB_COMMIT"
+    git -C $FB_DIR reset --hard $FB_COMMIT
+
+    pushd $FB_DIR  
+    echo "run: bazel build :flatc ..."
+    bazel build :flatc
+    popd
+fi
+
 # Execute the code generation:
-flatc --rust -o rust/arrow/src/ipc/gen/ format/*.fbs
+$FLATC --rust -o rust/arrow/src/ipc/gen/ format/*.fbs
 
 # Now the files are wrongly named so we have to change that.
 popd
@@ -63,6 +98,8 @@ SCHEMA_IMPORT="\nuse crate::ipc::gen::Schema::*;"
 SPARSE_TENSOR_IMPORT="\nuse crate::ipc::gen::SparseTensor::*;"
 TENSOR_IMPORT="\nuse crate::ipc::gen::Tensor::*;"
 
+# For flatbuffer(1.12.0+), remove: use crate::${name}_generated::\*;
+names=("File" "Message" "Schema" "SparseTensor" "Tensor")
 
 # Remove all generated lines we don't need
 for f in `ls *.rs`; do
@@ -86,6 +123,12 @@ for f in `ls *.rs`; do
     sed -i '' '/use std::mem;/d' $f
     sed -i '' '/use std::cmp::Ordering;/d' $f
 
+    # required by flatc 1.12.0+
+    sed -i '' "/\#\!\[allow(unused_imports, dead_code)\]/d" $f
+    for name in ${names[@]}; do
+        sed -i '' "/use crate::${name}_generated::\*;/d" $f
+    done
+
     # Replace all occurrences of type__ with type_
     sed -i '' 's/type__/type_/g' $f
 
@@ -106,4 +149,19 @@ done
 
 # Return back to base directory
 popd
-cargo +stable fmt -- src/ipc/gen/*
+cargo +stable fmt -- src/ipc/gen/*
+
+echo "=== TIPS ==="
+echo "Let's manually fix rustdoc of SparseTensorIndexCSF::indptrType:"
+echo 'prepend the tree with ```text, and append the tree with ```'
+cat <<TREE_EOF
+    /// \`\`\`text
+    ///         0          1
+    ///        / \         |
+    ///       0   1        1
+    ///      /   / \       |
+    ///     0   0   1      1
+    ///    /|  /|   |    /| |
+    ///   1 2 0 2   0   0 1 2
+    /// \`\`\`
+TREE_EOF

rust/arrow/src/ipc/convert.rs

@@ -166,7 +166,10 @@ pub(crate) fn get_data_type(field: ipc::Field, may_be_dictionary: bool) -> DataT
                 (32, false) => DataType::UInt32,
                 (64, true) => DataType::Int64,
                 (64, false) => DataType::UInt64,
-                _ => panic!("Unexpected bitwidth and signed"),
+                z => panic!(
+                    "Int type with bit width of {} and signed of {} not supported",
+                    z.0, z.1
+                ),
             }
         }
         ipc::Type::Binary => DataType::Binary,
@@ -183,13 +186,15 @@ pub(crate) fn get_data_type(field: ipc::Field, may_be_dictionary: bool) -> DataT
                 ipc::Precision::HALF => DataType::Float16,
                 ipc::Precision::SINGLE => DataType::Float32,
                 ipc::Precision::DOUBLE => DataType::Float64,
+                z => panic!("FloatingPoint type with precision of {:?} not supported", z),
             }
         }
         ipc::Type::Date => {
             let date = field.type_as_date().unwrap();
             match date.unit() {
                 ipc::DateUnit::DAY => DataType::Date32(DateUnit::Day),
                 ipc::DateUnit::MILLISECOND => DataType::Date64(DateUnit::Millisecond),
+                z => panic!("Date type with unit of {:?} not supported", z),
             }
         }
         ipc::Type::Time => {
@@ -224,6 +229,7 @@ pub(crate) fn get_data_type(field: ipc::Field, may_be_dictionary: bool) -> DataT
                 ipc::TimeUnit::NANOSECOND => {
                     DataType::Timestamp(TimeUnit::Nanosecond, timezone)
                 }
+                z => panic!("Timestamp type with unit of {:?} not supported", z),
             }
         }
         ipc::Type::Interval => {
@@ -233,6 +239,7 @@ pub(crate) fn get_data_type(field: ipc::Field, may_be_dictionary: bool) -> DataT
                     DataType::Interval(IntervalUnit::YearMonth)
                 }
                 ipc::IntervalUnit::DAY_TIME => DataType::Interval(IntervalUnit::DayTime),
+                z => panic!("Interval type with unit of {:?} unsupported", z),
             }
         }
         ipc::Type::Duration => {
@@ -242,6 +249,7 @@ pub(crate) fn get_data_type(field: ipc::Field, may_be_dictionary: bool) -> DataT
                 ipc::TimeUnit::MILLISECOND => DataType::Duration(TimeUnit::Millisecond),
                 ipc::TimeUnit::MICROSECOND => DataType::Duration(TimeUnit::Microsecond),
                 ipc::TimeUnit::NANOSECOND => DataType::Duration(TimeUnit::Nanosecond),
+                z => panic!("Duration type with unit of {:?} unsupported", z),
             }
         }
         ipc::Type::List => {

rust/arrow/src/ipc/gen/File.rs

@@ -25,13 +25,23 @@ use std::{cmp::Ordering, mem};
 
 // struct Block, aligned to 8
 #[repr(C, align(8))]
-#[derive(Clone, Copy, Debug, PartialEq)]
+#[derive(Clone, Copy, PartialEq)]
 pub struct Block {
     offset_: i64,
     metaDataLength_: i32,
     padding0__: u32,
     bodyLength_: i64,
 } // pub struct Block
+impl std::fmt::Debug for Block {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        f.debug_struct("Block")
+            .field("offset", &self.offset())
+            .field("metaDataLength", &self.metaDataLength())
+            .field("bodyLength", &self.bodyLength())
+            .finish()
+    }
+}
+
 impl flatbuffers::SafeSliceAccess for Block {}
 impl<'a> flatbuffers::Follow<'a> for Block {
     type Inner = &'a Block;
@@ -52,7 +62,7 @@ impl<'b> flatbuffers::Push for Block {
     #[inline]
     fn push(&self, dst: &mut [u8], _rest: &[u8]) {
         let src = unsafe {
-            std::slice::from_raw_parts(self as *const Block as *const u8, Self::size())
+            ::std::slice::from_raw_parts(self as *const Block as *const u8, Self::size())
         };
         dst.copy_from_slice(src);
     }
@@ -63,14 +73,14 @@ impl<'b> flatbuffers::Push for &'b Block {
     #[inline]
     fn push(&self, dst: &mut [u8], _rest: &[u8]) {
         let src = unsafe {
-            std::slice::from_raw_parts(*self as *const Block as *const u8, Self::size())
+            ::std::slice::from_raw_parts(*self as *const Block as *const u8, Self::size())
         };
         dst.copy_from_slice(src);
     }
 }
 
 impl Block {
-    pub fn new<'a>(_offset: i64, _metaDataLength: i32, _bodyLength: i64) -> Self {
+    pub fn new(_offset: i64, _metaDataLength: i32, _bodyLength: i64) -> Self {
         Block {
             offset_: _offset.to_little_endian(),
             metaDataLength_: _metaDataLength.to_little_endian(),
@@ -80,22 +90,22 @@ impl Block {
         }
     }
     /// Index to the start of the RecordBlock (note this is past the Message header)
-    pub fn offset<'a>(&'a self) -> i64 {
+    pub fn offset(&self) -> i64 {
         self.offset_.from_little_endian()
     }
     /// Length of the metadata
-    pub fn metaDataLength<'a>(&'a self) -> i32 {
+    pub fn metaDataLength(&self) -> i32 {
         self.metaDataLength_.from_little_endian()
     }
     /// Length of the data (this is aligned so there can be a gap between this and
     /// the metadata).
-    pub fn bodyLength<'a>(&'a self) -> i64 {
+    pub fn bodyLength(&self) -> i64 {
         self.bodyLength_.from_little_endian()
     }
 }
 
 pub enum FooterOffset {}
-#[derive(Copy, Clone, Debug, PartialEq)]
+#[derive(Copy, Clone, PartialEq)]
 
 /// ----------------------------------------------------------------------
 /// Arrow File metadata
@@ -109,7 +119,7 @@ impl<'a> flatbuffers::Follow<'a> for Footer<'a> {
     #[inline]
     fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
         Self {
-            _tab: flatbuffers::Table { buf: buf, loc: loc },
+            _tab: flatbuffers::Table { buf, loc },
         }
     }
 }
@@ -280,6 +290,17 @@ impl<'a: 'b, 'b> FooterBuilder<'a, 'b> {
     }
 }
 
+impl std::fmt::Debug for Footer<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut ds = f.debug_struct("Footer");
+        ds.field("version", &self.version());
+        ds.field("schema", &self.schema());
+        ds.field("dictionaries", &self.dictionaries());
+        ds.field("recordBatches", &self.recordBatches());
+        ds.field("custom_metadata", &self.custom_metadata());
+        ds.finish()
+    }
+}
 #[inline]
 pub fn get_root_as_footer<'a>(buf: &'a [u8]) -> Footer<'a> {
     flatbuffers::get_root::<Footer<'a>>(buf)