From e836c965362f0436ec752b4b0588ba3ba94b876b Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Thu, 9 Apr 2020 10:49:46 -0700 Subject: [PATCH 01/13] initial avro parser --- .../storage/blob/_shared/avro/__init__.py | 0 .../storage/blob/_shared/avro/datafile.py | 301 ++++ .../azure/storage/blob/_shared/avro/io.py | 657 +++++++++ .../azure/storage/blob/_shared/avro/schema.py | 1252 +++++++++++++++++ .../tests/_shared/avro/__init__.py | 0 .../_shared/avro/samples/changeFeed.avro | Bin 0 -> 2514 bytes .../_shared/avro/samples/test_deflate_0.avro | Bin 0 -> 83 bytes .../_shared/avro/samples/test_deflate_1.avro | Bin 0 -> 89 bytes .../_shared/avro/samples/test_deflate_10.avro | Bin 0 -> 119 bytes .../_shared/avro/samples/test_deflate_11.avro | Bin 0 -> 124 bytes .../_shared/avro/samples/test_deflate_12.avro | Bin 0 -> 106 bytes .../_shared/avro/samples/test_deflate_13.avro | Bin 0 -> 158 bytes .../_shared/avro/samples/test_deflate_14.avro | Bin 0 -> 296 bytes .../_shared/avro/samples/test_deflate_2.avro | Bin 0 -> 108 bytes .../_shared/avro/samples/test_deflate_3.avro | Bin 0 -> 97 bytes .../_shared/avro/samples/test_deflate_4.avro | Bin 0 -> 87 bytes .../_shared/avro/samples/test_deflate_5.avro | Bin 0 -> 88 bytes .../_shared/avro/samples/test_deflate_6.avro | Bin 0 -> 91 bytes .../_shared/avro/samples/test_deflate_7.avro | Bin 0 -> 94 bytes .../_shared/avro/samples/test_deflate_8.avro | Bin 0 -> 124 bytes .../_shared/avro/samples/test_deflate_9.avro | Bin 0 -> 135 bytes .../_shared/avro/samples/test_null_0.avro | Bin 0 -> 75 bytes .../_shared/avro/samples/test_null_1.avro | Bin 0 -> 88 bytes .../_shared/avro/samples/test_null_10.avro | Bin 0 -> 153 bytes .../_shared/avro/samples/test_null_11.avro | Bin 0 -> 213 bytes .../_shared/avro/samples/test_null_12.avro | Bin 0 -> 105 bytes .../_shared/avro/samples/test_null_13.avro | Bin 0 -> 157 bytes .../_shared/avro/samples/test_null_14.avro | Bin 0 -> 358 bytes .../_shared/avro/samples/test_null_2.avro | Bin 0 -> 308 bytes .../_shared/avro/samples/test_null_3.avro | Bin 0 -> 177 bytes .../_shared/avro/samples/test_null_4.avro | Bin 0 -> 94 bytes .../_shared/avro/samples/test_null_5.avro | Bin 0 -> 95 bytes .../_shared/avro/samples/test_null_6.avro | Bin 0 -> 116 bytes .../_shared/avro/samples/test_null_7.avro | Bin 0 -> 158 bytes .../_shared/avro/samples/test_null_8.avro | Bin 0 -> 123 bytes .../_shared/avro/samples/test_null_9.avro | Bin 0 -> 134 bytes .../tests/_shared/avro/test_avro.py | 110 ++ 37 files changed, 2320 insertions(+) create mode 100644 sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/__init__.py create mode 100644 sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py create mode 100644 sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/io.py create mode 100644 sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/__init__.py create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/changeFeed.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_0.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_1.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_10.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_11.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_12.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_13.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_14.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_2.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_3.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_4.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_5.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_6.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_7.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_8.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_9.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_0.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_1.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_10.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_11.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_12.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_13.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_14.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_2.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_3.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_4.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_5.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_6.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_7.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_8.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_9.avro create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/__init__.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py new file mode 100644 index 000000000000..2da8c896c2f8 --- /dev/null +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Read/Write Avro File Object Containers.""" + +import io +import logging +import zlib + +from ..avro import io as avro_io +from ..avro import schema + + +logger = logging.getLogger(__name__) + +# ------------------------------------------------------------------------------ +# Constants + +# Version of the container file: +VERSION = 1 + +# Magic code that starts a data container file: +MAGIC = b'Obj' + bytes([VERSION]) + +# Size of the magic code, in number of bytes: +MAGIC_SIZE = len(MAGIC) + +# Size of the synchronization marker, in number of bytes: +SYNC_SIZE = 16 + +# Interval between synchronization markers, in number of bytes: +# TODO: make configurable +SYNC_INTERVAL = 1000 * SYNC_SIZE + +# Schema of the container header: +META_SCHEMA = schema.parse(""" +{ + "type": "record", "name": "org.apache.avro.file.Header", + "fields": [{ + "name": "magic", + "type": {"type": "fixed", "name": "magic", "size": %(magic_size)d} + }, { + "name": "meta", + "type": {"type": "map", "values": "bytes"} + }, { + "name": "sync", + "type": {"type": "fixed", "name": "sync", "size": %(sync_size)d} + }] +} +""" % { + 'magic_size': MAGIC_SIZE, + 'sync_size': SYNC_SIZE, +}) + +# Codecs supported by container files: +VALID_CODECS = frozenset(['null', 'deflate']) + +# Not used yet +VALID_ENCODINGS = frozenset(['binary']) + +# Metadata key associated to the codec: +CODEC_KEY = "avro.codec" + +# Metadata key associated to the schema: +SCHEMA_KEY = "avro.schema" + + +# ------------------------------------------------------------------------------ +# Exceptions + + +class DataFileException(schema.AvroException): + """Problem reading or writing file object containers.""" + + def __init__(self, msg): + super(DataFileException, self).__init__(msg) + +# ------------------------------------------------------------------------------ + + +class DataFileReader(object): + """Read files written by DataFileWriter.""" + + # TODO: allow user to specify expected schema? + # TODO: allow user to specify the encoder + def __init__(self, reader, datum_reader): + """Initializes a new data file reader. + + Args: + reader: Open file to read from. + datum_reader: Avro datum reader. + """ + self._reader = reader + self._raw_decoder = avro_io.BinaryDecoder(reader) + self._datum_decoder = None # Maybe reset at every block. + self._datum_reader = datum_reader + + # read the header: magic, meta, sync + self._read_header() + + # ensure codec is valid + avro_codec_raw = self.GetMeta('avro.codec') + if avro_codec_raw is None: + self.codec = "null" + else: + self.codec = avro_codec_raw.decode('utf-8') + if self.codec not in VALID_CODECS: + raise DataFileException('Unknown codec: %s.' % self.codec) + + self._file_length = self._GetInputFileLength() + + # get ready to read + self._block_count = 0 + self.datum_reader.writer_schema = ( + schema.parse(self.GetMeta(SCHEMA_KEY).decode('utf-8'))) + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + # Perform a close if there's no exception + if type is None: + self.close() + + def __iter__(self): + return self + + def __next__(self): + """Implements the iterator interface.""" + return next(self) + + # read-only properties + @property + def reader(self): + return self._reader + + @property + def raw_decoder(self): + return self._raw_decoder + + @property + def datum_decoder(self): + return self._datum_decoder + + @property + def datum_reader(self): + return self._datum_reader + + @property + def sync_marker(self): + return self._sync_marker + + @property + def meta(self): + return self._meta + + @property + def file_length(self): + """Length of the input file, in bytes.""" + return self._file_length + + # read/write properties + @property + def block_count(self): + return self._block_count + + def GetMeta(self, key): + """Reports the value of a given metadata key. + + Args: + key: Metadata key (string) to report the value of. + Returns: + Value associated to the metadata key, as bytes. + """ + return self._meta.get(key) + + def SetMeta(self, key, value): + """Sets a metadata. + + Args: + key: Metadata key (string) to set. + value: Metadata value to set, as bytes. + """ + if isinstance(value, str): + value = value.encode('utf-8') + self._meta[key] = value + + def _GetInputFileLength(self): + """Reports the length of the input file, in bytes. + + Leaves the current position unmodified. + + Returns: + The length of the input file, in bytes. + """ + current_pos = self.reader.tell() + self.reader.seek(0, 2) + file_length = self.reader.tell() + self.reader.seek(current_pos) + return file_length + + def is_EOF(self): + return self.reader.tell() == self.file_length + + def _read_header(self): + # seek to the beginning of the file to get magic block + self.reader.seek(0, 0) + + # read header into a dict + header = self.datum_reader.read_data( + META_SCHEMA, META_SCHEMA, self.raw_decoder) + + # check magic number + if header.get('magic') != MAGIC: + fail_msg = "Not an Avro data file: %s doesn't match %s."\ + % (header.get('magic'), MAGIC) + raise schema.AvroException(fail_msg) + + # set metadata + self._meta = header['meta'] + + # set sync marker + self._sync_marker = header['sync'] + + def _read_block_header(self): + self._block_count = self.raw_decoder.read_long() + if self.codec == "null": + # Skip a long; we don't need to use the length. + self.raw_decoder.skip_long() + self._datum_decoder = self._raw_decoder + elif self.codec == 'deflate': + # Compressed data is stored as (length, data), which + # corresponds to how the "bytes" type is encoded. + data = self.raw_decoder.read_bytes() + # -15 is the log of the window size; negative indicates + # "raw" (no zlib headers) decompression. See zlib.h. + uncompressed = zlib.decompress(data, -15) + self._datum_decoder = avro_io.BinaryDecoder(io.BytesIO(uncompressed)) + elif self.codec == 'snappy': + # Compressed data includes a 4-byte CRC32 checksum + length = self.raw_decoder.read_long() + data = self.raw_decoder.read(length - 4) + uncompressed = snappy.decompress(data) + self._datum_decoder = avro_io.BinaryDecoder(io.BytesIO(uncompressed)) + self.raw_decoder.check_crc32(uncompressed); + else: + raise DataFileException("Unknown codec: %r" % self.codec) + + def _skip_sync(self): + """ + Read the length of the sync marker; if it matches the sync marker, + return True. Otherwise, seek back to where we started and return False. + """ + proposed_sync_marker = self.reader.read(SYNC_SIZE) + if proposed_sync_marker != self.sync_marker: + self.reader.seek(-SYNC_SIZE, 1) + return False + else: + return True + + # TODO: handle block of length zero + # TODO: clean this up with recursion + def __next__(self): + """Return the next datum in the file.""" + if self.block_count == 0: + if self.is_EOF(): + raise StopIteration + elif self._skip_sync(): + if self.is_EOF(): raise StopIteration + self._read_block_header() + else: + self._read_block_header() + + datum = self.datum_reader.read(self.datum_decoder) + self._block_count -= 1 + return datum + + def close(self): + """Close this reader.""" + self.reader.close() + + +if __name__ == '__main__': + raise Exception('Not a standalone module') diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/io.py new file mode 100644 index 000000000000..48e795386f77 --- /dev/null +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/io.py @@ -0,0 +1,657 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Input/output utilities. + +Includes: + - i/o-specific constants + - i/o-specific exceptions + - schema validation + - leaf value encoding and decoding + - datum reader/writer stuff (?) + +Also includes a generic representation for data, which uses the +following mapping: + - Schema records are implemented as dict. + - Schema arrays are implemented as list. + - Schema maps are implemented as dict. + - Schema strings are implemented as unicode. + - Schema bytes are implemented as str. + - Schema ints are implemented as int. + - Schema longs are implemented as long. + - Schema floats are implemented as float. + - Schema doubles are implemented as float. + - Schema booleans are implemented as bool. +""" + +import binascii +import json +import logging +import struct + +from ..avro import schema + +logger = logging.getLogger(__name__) + +# ------------------------------------------------------------------------------ +# Constants + + +INT_MIN_VALUE = -(1 << 31) +INT_MAX_VALUE = (1 << 31) - 1 +LONG_MIN_VALUE = -(1 << 63) +LONG_MAX_VALUE = (1 << 63) - 1 + +STRUCT_INT = struct.Struct('!I') # big-endian unsigned int +STRUCT_FLOAT = struct.Struct('I') # big-endian unsigned int + + +# ------------------------------------------------------------------------------ +# Exceptions + + +class AvroTypeException(schema.AvroException): + """Raised when datum is not an example of schema.""" + def __init__(self, expected_schema, datum): + pretty_expected = json.dumps(json.loads(str(expected_schema)), indent=2) + fail_msg = "The datum %s is not an example of the schema %s"\ + % (datum, pretty_expected) + schema.AvroException.__init__(self, fail_msg) + + +class SchemaResolutionException(schema.AvroException): + def __init__(self, fail_msg, writer_schema=None, reader_schema=None): + pretty_writers = json.dumps(json.loads(str(writer_schema)), indent=2) + pretty_readers = json.dumps(json.loads(str(reader_schema)), indent=2) + if writer_schema: fail_msg += "\nWriter's Schema: %s" % pretty_writers + if reader_schema: fail_msg += "\nReader's Schema: %s" % pretty_readers + schema.AvroException.__init__(self, fail_msg) + + +# ------------------------------------------------------------------------------ +# Validate + +_valid = { + 'null': lambda s, d: d is None, + 'boolean': lambda s, d: isinstance(d, bool), + 'string': lambda s, d: isinstance(d, str), + 'bytes': lambda s, d: isinstance(d, bytes), + 'int': lambda s, d: isinstance(d, int) and (INT_MIN_VALUE <= d <= INT_MAX_VALUE), + 'long': lambda s, d: isinstance(d, int) and (LONG_MIN_VALUE <= d <= LONG_MAX_VALUE), + 'float': lambda s, d: isinstance(d, (int, float)), + 'fixed': lambda s, d: isinstance(d, bytes) and len(d) == s.size, + 'enum': lambda s, d: d in s.symbols, + 'array': lambda s, d: isinstance(d, list) and all(Validate(s.items, item) for item in d), + 'map': lambda s, d: (isinstance(d, dict) and all(isinstance(key, str) for key in d) + and all(Validate(s.values, value) for value in d.values())), + 'union': lambda s, d: any(Validate(branch, d) for branch in s.schemas), + 'record': lambda s, d: (isinstance(d, dict) + and all(Validate(f.type, d.get(f.name)) for f in s.fields) + and {f.name for f in s.fields}.issuperset(d.keys())) +} +_valid['double'] = _valid['float'] +_valid['error_union'] = _valid['union'] +_valid['error'] = _valid['request'] = _valid['record'] + +def Validate(expected_schema, datum): + """Determines if a python datum is an instance of a schema. + + Args: + expected_schema: Schema to validate against. + datum: Datum to validate. + Returns: + True if the datum is an instance of the schema. + """ + try: + return _valid[expected_schema.type](expected_schema, datum) + except KeyError: + raise AvroTypeException(expected_schema, datum) + + +# ------------------------------------------------------------------------------ +# Decoder/Encoder + + +class BinaryDecoder(object): + """Read leaf values.""" + def __init__(self, reader): + """ + reader is a Python object on which we can call read, seek, and tell. + """ + self._reader = reader + + @property + def reader(self): + """Reports the reader used by this decoder.""" + return self._reader + + def read(self, n): + """Read n bytes. + + Args: + n: Number of bytes to read. + Returns: + The next n bytes from the input. + """ + assert (n >= 0), n + input_bytes = self.reader.read(n) + assert (len(input_bytes) == n), input_bytes + return input_bytes + + def read_null(self): + """ + null is written as zero bytes + """ + return None + + def read_boolean(self): + """ + a boolean is written as a single byte + whose value is either 0 (false) or 1 (true). + """ + return ord(self.read(1)) == 1 + + def read_int(self): + """ + int and long values are written using variable-length, zig-zag coding. + """ + return self.read_long() + + def read_long(self): + """ + int and long values are written using variable-length, zig-zag coding. + """ + b = ord(self.read(1)) + n = b & 0x7F + shift = 7 + while (b & 0x80) != 0: + b = ord(self.read(1)) + n |= (b & 0x7F) << shift + shift += 7 + datum = (n >> 1) ^ -(n & 1) + return datum + + def read_float(self): + """ + A float is written as 4 bytes. + The float is converted into a 32-bit integer using a method equivalent to + Java's floatToIntBits and then encoded in little-endian format. + """ + return STRUCT_FLOAT.unpack(self.read(4))[0] + + def read_double(self): + """ + A double is written as 8 bytes. + The double is converted into a 64-bit integer using a method equivalent to + Java's doubleToLongBits and then encoded in little-endian format. + """ + return STRUCT_DOUBLE.unpack(self.read(8))[0] + + def read_bytes(self): + """ + Bytes are encoded as a long followed by that many bytes of data. + """ + nbytes = self.read_long() + assert (nbytes >= 0), nbytes + return self.read(nbytes) + + def read_utf8(self): + """ + A string is encoded as a long followed by + that many bytes of UTF-8 encoded character data. + """ + input_bytes = self.read_bytes() + try: + return input_bytes.decode('utf-8') + except UnicodeDecodeError as exn: + logger.error('Invalid UTF-8 input bytes: %r', input_bytes) + raise exn + + def check_crc32(self, bytes): + checksum = STRUCT_CRC32.unpack(self.read(4))[0]; + if binascii.crc32(bytes) & 0xffffffff != checksum: + raise schema.AvroException("Checksum failure") + + def skip_null(self): + pass + + def skip_boolean(self): + self.skip(1) + + def skip_int(self): + self.skip_long() + + def skip_long(self): + b = ord(self.read(1)) + while (b & 0x80) != 0: + b = ord(self.read(1)) + + def skip_float(self): + self.skip(4) + + def skip_double(self): + self.skip(8) + + def skip_bytes(self): + self.skip(self.read_long()) + + def skip_utf8(self): + self.skip_bytes() + + def skip(self, n): + self.reader.seek(self.reader.tell() + n) + +# ------------------------------------------------------------------------------ +# DatumReader/Writer + + +class DatumReader(object): + """Deserialize Avro-encoded data into a Python data structure.""" + @staticmethod + def check_props(schema_one, schema_two, prop_list): + for prop in prop_list: + if getattr(schema_one, prop) != getattr(schema_two, prop): + return False + return True + + @staticmethod + def match_schemas(writer_schema, reader_schema): + w_type = writer_schema.type + r_type = reader_schema.type + if 'union' in [w_type, r_type] or 'error_union' in [w_type, r_type]: + return True + elif (w_type in schema.PRIMITIVE_TYPES and r_type in schema.PRIMITIVE_TYPES + and w_type == r_type): + return True + elif (w_type == r_type == 'record' and + DatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + return True + elif (w_type == r_type == 'error' and + DatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + return True + elif (w_type == r_type == 'request'): + return True + elif (w_type == r_type == 'fixed' and + DatumReader.check_props(writer_schema, reader_schema, + ['fullname', 'size'])): + return True + elif (w_type == r_type == 'enum' and + DatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + return True + elif (w_type == r_type == 'map' and + DatumReader.check_props(writer_schema.values, + reader_schema.values, ['type'])): + return True + elif (w_type == r_type == 'array' and + DatumReader.check_props(writer_schema.items, + reader_schema.items, ['type'])): + return True + + # Handle schema promotion + if w_type == 'int' and r_type in ['long', 'float', 'double']: + return True + elif w_type == 'long' and r_type in ['float', 'double']: + return True + elif w_type == 'float' and r_type == 'double': + return True + return False + + def __init__(self, writer_schema=None, reader_schema=None): + """ + As defined in the Avro specification, we call the schema encoded + in the data the "writer's schema", and the schema expected by the + reader the "reader's schema". + """ + self._writer_schema = writer_schema + self._reader_schema = reader_schema + + # read/write properties + def set_writer_schema(self, writer_schema): + self._writer_schema = writer_schema + writer_schema = property(lambda self: self._writer_schema, + set_writer_schema) + def set_reader_schema(self, reader_schema): + self._reader_schema = reader_schema + reader_schema = property(lambda self: self._reader_schema, + set_reader_schema) + + def read(self, decoder): + if self.reader_schema is None: + self.reader_schema = self.writer_schema + return self.read_data(self.writer_schema, self.reader_schema, decoder) + + def read_data(self, writer_schema, reader_schema, decoder): + # schema matching + if not DatumReader.match_schemas(writer_schema, reader_schema): + fail_msg = 'Schemas do not match.' + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + # schema resolution: reader's schema is a union, writer's schema is not + if (writer_schema.type not in ['union', 'error_union'] + and reader_schema.type in ['union', 'error_union']): + for s in reader_schema.schemas: + if DatumReader.match_schemas(writer_schema, s): + return self.read_data(writer_schema, s, decoder) + fail_msg = 'Schemas do not match.' + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + # function dispatch for reading data based on type of writer's schema + if writer_schema.type == 'null': + return decoder.read_null() + elif writer_schema.type == 'boolean': + return decoder.read_boolean() + elif writer_schema.type == 'string': + return decoder.read_utf8() + elif writer_schema.type == 'int': + return decoder.read_int() + elif writer_schema.type == 'long': + return decoder.read_long() + elif writer_schema.type == 'float': + return decoder.read_float() + elif writer_schema.type == 'double': + return decoder.read_double() + elif writer_schema.type == 'bytes': + return decoder.read_bytes() + elif writer_schema.type == 'fixed': + return self.read_fixed(writer_schema, reader_schema, decoder) + elif writer_schema.type == 'enum': + return self.read_enum(writer_schema, reader_schema, decoder) + elif writer_schema.type == 'array': + return self.read_array(writer_schema, reader_schema, decoder) + elif writer_schema.type == 'map': + return self.read_map(writer_schema, reader_schema, decoder) + elif writer_schema.type in ['union', 'error_union']: + return self.read_union(writer_schema, reader_schema, decoder) + elif writer_schema.type in ['record', 'error', 'request']: + return self.read_record(writer_schema, reader_schema, decoder) + else: + fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type + raise schema.AvroException(fail_msg) + + def skip_data(self, writer_schema, decoder): + if writer_schema.type == 'null': + return decoder.skip_null() + elif writer_schema.type == 'boolean': + return decoder.skip_boolean() + elif writer_schema.type == 'string': + return decoder.skip_utf8() + elif writer_schema.type == 'int': + return decoder.skip_int() + elif writer_schema.type == 'long': + return decoder.skip_long() + elif writer_schema.type == 'float': + return decoder.skip_float() + elif writer_schema.type == 'double': + return decoder.skip_double() + elif writer_schema.type == 'bytes': + return decoder.skip_bytes() + elif writer_schema.type == 'fixed': + return self.skip_fixed(writer_schema, decoder) + elif writer_schema.type == 'enum': + return self.skip_enum(writer_schema, decoder) + elif writer_schema.type == 'array': + return self.skip_array(writer_schema, decoder) + elif writer_schema.type == 'map': + return self.skip_map(writer_schema, decoder) + elif writer_schema.type in ['union', 'error_union']: + return self.skip_union(writer_schema, decoder) + elif writer_schema.type in ['record', 'error', 'request']: + return self.skip_record(writer_schema, decoder) + else: + fail_msg = "Unknown schema type: %s" % writer_schema.type + raise schema.AvroException(fail_msg) + + def read_fixed(self, writer_schema, reader_schema, decoder): + """ + Fixed instances are encoded using the number of bytes declared + in the schema. + """ + return decoder.read(writer_schema.size) + + def skip_fixed(self, writer_schema, decoder): + return decoder.skip(writer_schema.size) + + def read_enum(self, writer_schema, reader_schema, decoder): + """ + An enum is encoded by a int, representing the zero-based position + of the symbol in the schema. + """ + # read data + index_of_symbol = decoder.read_int() + if index_of_symbol >= len(writer_schema.symbols): + fail_msg = "Can't access enum index %d for enum with %d symbols"\ + % (index_of_symbol, len(writer_schema.symbols)) + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + read_symbol = writer_schema.symbols[index_of_symbol] + + # schema resolution + if read_symbol not in reader_schema.symbols: + fail_msg = "Symbol %s not present in Reader's Schema" % read_symbol + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + return read_symbol + + def skip_enum(self, writer_schema, decoder): + return decoder.skip_int() + + def read_array(self, writer_schema, reader_schema, decoder): + """ + Arrays are encoded as a series of blocks. + + Each block consists of a long count value, + followed by that many array items. + A block with count zero indicates the end of the array. + Each item is encoded per the array's item schema. + + If a block's count is negative, + then the count is followed immediately by a long block size, + indicating the number of bytes in the block. + The actual count in this case + is the absolute value of the count written. + """ + read_items = [] + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_count = -block_count + block_size = decoder.read_long() + for i in range(block_count): + read_items.append(self.read_data(writer_schema.items, + reader_schema.items, decoder)) + block_count = decoder.read_long() + return read_items + + def skip_array(self, writer_schema, decoder): + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_size = decoder.read_long() + decoder.skip(block_size) + else: + for i in range(block_count): + self.skip_data(writer_schema.items, decoder) + block_count = decoder.read_long() + + def read_map(self, writer_schema, reader_schema, decoder): + """ + Maps are encoded as a series of blocks. + + Each block consists of a long count value, + followed by that many key/value pairs. + A block with count zero indicates the end of the map. + Each item is encoded per the map's value schema. + + If a block's count is negative, + then the count is followed immediately by a long block size, + indicating the number of bytes in the block. + The actual count in this case + is the absolute value of the count written. + """ + read_items = {} + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_count = -block_count + block_size = decoder.read_long() + for i in range(block_count): + key = decoder.read_utf8() + read_items[key] = self.read_data(writer_schema.values, + reader_schema.values, decoder) + block_count = decoder.read_long() + return read_items + + def skip_map(self, writer_schema, decoder): + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_size = decoder.read_long() + decoder.skip(block_size) + else: + for i in range(block_count): + decoder.skip_utf8() + self.skip_data(writer_schema.values, decoder) + block_count = decoder.read_long() + + def read_union(self, writer_schema, reader_schema, decoder): + """ + A union is encoded by first writing a long value indicating + the zero-based position within the union of the schema of its value. + The value is then encoded per the indicated schema within the union. + """ + # schema resolution + index_of_schema = int(decoder.read_long()) + if index_of_schema >= len(writer_schema.schemas): + fail_msg = "Can't access branch index %d for union with %d branches"\ + % (index_of_schema, len(writer_schema.schemas)) + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + selected_writer_schema = writer_schema.schemas[index_of_schema] + + # read data + return self.read_data(selected_writer_schema, reader_schema, decoder) + + def skip_union(self, writer_schema, decoder): + index_of_schema = int(decoder.read_long()) + if index_of_schema >= len(writer_schema.schemas): + fail_msg = "Can't access branch index %d for union with %d branches"\ + % (index_of_schema, len(writer_schema.schemas)) + raise SchemaResolutionException(fail_msg, writer_schema) + return self.skip_data(writer_schema.schemas[index_of_schema], decoder) + + def read_record(self, writer_schema, reader_schema, decoder): + """ + A record is encoded by encoding the values of its fields + in the order that they are declared. In other words, a record + is encoded as just the concatenation of the encodings of its fields. + Field values are encoded per their schema. + + Schema Resolution: + * the ordering of fields may be different: fields are matched by name. + * schemas for fields with the same name in both records are resolved + recursively. + * if the writer's record contains a field with a name not present in the + reader's record, the writer's value for that field is ignored. + * if the reader's record schema has a field that contains a default value, + and writer's schema does not have a field with the same name, then the + reader should use the default value from its field. + * if the reader's record schema has a field with no default value, and + writer's schema does not have a field with the same name, then the + field's value is unset. + """ + # schema resolution + readers_fields_dict = reader_schema.field_map + read_record = {} + for field in writer_schema.fields: + readers_field = readers_fields_dict.get(field.name) + if readers_field is not None: + field_val = self.read_data(field.type, readers_field.type, decoder) + read_record[field.name] = field_val + else: + self.skip_data(field.type, decoder) + + # fill in default values + if len(readers_fields_dict) > len(read_record): + writers_fields_dict = writer_schema.field_map + for field_name, field in readers_fields_dict.items(): + if field_name not in writers_fields_dict: + if field.has_default: + field_val = self._read_default_value(field.type, field.default) + read_record[field.name] = field_val + else: + fail_msg = 'No default value for field %s' % field_name + raise SchemaResolutionException(fail_msg, writer_schema, + reader_schema) + return read_record + + def skip_record(self, writer_schema, decoder): + for field in writer_schema.fields: + self.skip_data(field.type, decoder) + + def _read_default_value(self, field_schema, default_value): + """ + Basically a JSON Decoder? + """ + if field_schema.type == 'null': + return None + elif field_schema.type == 'boolean': + return bool(default_value) + elif field_schema.type == 'int': + return int(default_value) + elif field_schema.type == 'long': + return int(default_value) + elif field_schema.type in ['float', 'double']: + return float(default_value) + elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']: + return default_value + elif field_schema.type == 'array': + read_array = [] + for json_val in default_value: + item_val = self._read_default_value(field_schema.items, json_val) + read_array.append(item_val) + return read_array + elif field_schema.type == 'map': + read_map = {} + for key, json_val in default_value.items(): + map_val = self._read_default_value(field_schema.values, json_val) + read_map[key] = map_val + return read_map + elif field_schema.type in ['union', 'error_union']: + return self._read_default_value(field_schema.schemas[0], default_value) + elif field_schema.type == 'record': + read_record = {} + for field in field_schema.fields: + json_val = default_value.get(field.name) + if json_val is None: json_val = field.default + field_val = self._read_default_value(field.type, json_val) + read_record[field.name] = field_val + return read_record + else: + fail_msg = 'Unknown type: %s' % field_schema.type + raise schema.AvroException(fail_msg) + + +# ------------------------------------------------------------------------------ + +if __name__ == '__main__': + raise Exception('Not a standalone module') diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py new file mode 100644 index 000000000000..3f2260e85b49 --- /dev/null +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py @@ -0,0 +1,1252 @@ +#!/usr/bin/env python3 +# -*- mode: python -*- +# -*- coding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Representation of Avro schemas. + +A schema may be one of: + - A record, mapping field names to field value data; + - An error, equivalent to a record; + - An enum, containing one of a small set of symbols; + - An array of values, all of the same schema; + - A map containing string/value pairs, each of a declared schema; + - A union of other schemas; + - A fixed sized binary object; + - A unicode string; + - A sequence of bytes; + - A 32-bit signed int; + - A 64-bit signed long; + - A 32-bit floating-point float; + - A 64-bit floating-point double; + - A boolean; + - Null. +""" + +import abc +import json +import logging +import re +import warnings +from types import MappingProxyType + +logger = logging.getLogger(__name__) + +# ------------------------------------------------------------------------------ +# Constants + +# Log level more verbose than DEBUG=10, INFO=20, etc. +DEBUG_VERBOSE=5 + + +NULL = 'null' +BOOLEAN = 'boolean' +STRING = 'string' +BYTES = 'bytes' +INT = 'int' +LONG = 'long' +FLOAT = 'float' +DOUBLE = 'double' +FIXED = 'fixed' +ENUM = 'enum' +RECORD = 'record' +ERROR = 'error' +ARRAY = 'array' +MAP = 'map' +UNION = 'union' + +# Request and error unions are part of Avro protocols: +REQUEST = 'request' +ERROR_UNION = 'error_union' + +PRIMITIVE_TYPES = frozenset([ + NULL, + BOOLEAN, + STRING, + BYTES, + INT, + LONG, + FLOAT, + DOUBLE, +]) + +NAMED_TYPES = frozenset([ + FIXED, + ENUM, + RECORD, + ERROR, +]) + +VALID_TYPES = frozenset.union( + PRIMITIVE_TYPES, + NAMED_TYPES, + [ + ARRAY, + MAP, + UNION, + REQUEST, + ERROR_UNION, + ], +) + +SCHEMA_RESERVED_PROPS = frozenset([ + 'type', + 'name', + 'namespace', + 'fields', # Record + 'items', # Array + 'size', # Fixed + 'symbols', # Enum + 'values', # Map + 'doc', +]) + +FIELD_RESERVED_PROPS = frozenset([ + 'default', + 'name', + 'doc', + 'order', + 'type', +]) + +VALID_FIELD_SORT_ORDERS = frozenset([ + 'ascending', + 'descending', + 'ignore', +]) + + +# ------------------------------------------------------------------------------ +# Exceptions + + +class Error(Exception): + """Base class for errors in this module.""" + pass + + +class AvroException(Error): + """Generic Avro schema error.""" + pass + + +class SchemaParseException(AvroException): + """Error while parsing a JSON schema descriptor.""" + pass + +# ------------------------------------------------------------------------------ +# Utilities +class MappingProxyEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, MappingProxyType): + return obj.copy() + return json.JSONEncoder.default(self, obj) + +# ------------------------------------------------------------------------------ + + +class Schema(object, metaclass=abc.ABCMeta): + """Abstract base class for all Schema classes.""" + + def __init__(self, type, other_props=None): + """Initializes a new schema object. + + Args: + type: Type of the schema to initialize. + other_props: Optional dictionary of additional properties. + """ + if type not in VALID_TYPES: + raise SchemaParseException('%r is not a valid Avro type.' % type) + + # All properties of this schema, as a map: property name -> property value + self._props = {} + + self._props['type'] = type + self._type = type + + if other_props: + self._props.update(other_props) + + @property + def namespace(self): + """Returns: the namespace this schema belongs to, if any, or None.""" + return self._props.get('namespace', None) + + @property + def type(self): + """Returns: the type of this schema.""" + return self._type + + @property + def doc(self): + """Returns: the documentation associated to this schema, if any, or None.""" + return self._props.get('doc', None) + + @property + def props(self): + """Reports all the properties of this schema. + + Includes all properties, reserved and non reserved. + JSON properties of this schema are directly generated from this dict. + + Returns: + A read-only dictionary of properties associated to this schema. + """ + return MappingProxyType(self._props) + + @property + def other_props(self): + """Returns: the dictionary of non-reserved properties.""" + return dict(FilterKeysOut(items=self._props, keys=SCHEMA_RESERVED_PROPS)) + + def __str__(self): + """Returns: the JSON representation of this schema.""" + return json.dumps(self.to_json(), cls=MappingProxyEncoder) + + @abc.abstractmethod + def to_json(self, names): + """Converts the schema object into its AVRO specification representation. + + Schema types that have names (records, enums, and fixed) must + be aware of not re-defining schemas that are already listed + in the parameter names. + """ + raise Exception('Cannot run abstract method.') + + +# ------------------------------------------------------------------------------ + + +_RE_NAME = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') + +_RE_FULL_NAME = re.compile( + r'^' + r'[.]?(?:[A-Za-z_][A-Za-z0-9_]*[.])*' # optional namespace + r'([A-Za-z_][A-Za-z0-9_]*)' # name + r'$' +) + +class Name(object): + """Representation of an Avro name.""" + + def __init__(self, name, namespace=None): + """Parses an Avro name. + + Args: + name: Avro name to parse (relative or absolute). + namespace: Optional explicit namespace if the name is relative. + """ + # Normalize: namespace is always defined as a string, possibly empty. + if namespace is None: namespace = '' + + if '.' in name: + # name is absolute, namespace is ignored: + self._fullname = name + + match = _RE_FULL_NAME.match(self._fullname) + if match is None: + raise SchemaParseException( + 'Invalid absolute schema name: %r.' % self._fullname) + + self._name = match.group(1) + self._namespace = self._fullname[:-(len(self._name) + 1)] + + else: + # name is relative, combine with explicit namespace: + self._name = name + self._namespace = namespace + self._fullname = (self._name + if (not self._namespace) else + '%s.%s' % (self._namespace, self._name)) + + # Validate the fullname: + if _RE_FULL_NAME.match(self._fullname) is None: + raise SchemaParseException( + 'Invalid schema name %r infered from name %r and namespace %r.' + % (self._fullname, self._name, self._namespace)) + + def __eq__(self, other): + if not isinstance(other, Name): + return NotImplemented + return self.fullname == other.fullname + + @property + def simple_name(self): + """Returns: the simple name part of this name.""" + return self._name + + @property + def namespace(self): + """Returns: this name's namespace, possible the empty string.""" + return self._namespace + + @property + def fullname(self): + """Returns: the full name.""" + return self._fullname + + +# ------------------------------------------------------------------------------ + + +class Names(object): + """Tracks Avro named schemas and default namespace during parsing.""" + + def __init__(self, default_namespace=None, names=None): + """Initializes a new name tracker. + + Args: + default_namespace: Optional default namespace. + names: Optional initial mapping of known named schemas. + """ + if names is None: + names = {} + self._names = names + self._default_namespace = default_namespace + + @property + def names(self): + """Returns: the mapping of known named schemas.""" + return self._names + + @property + def default_namespace(self): + """Returns: the default namespace, if any, or None.""" + return self._default_namespace + + def NewWithDefaultNamespace(self, namespace): + """Creates a new name tracker from this tracker, but with a new default ns. + + Args: + namespace: New default namespace to use. + Returns: + New name tracker with the specified default namespace. + """ + return Names(names=self._names, default_namespace=namespace) + + def GetName(self, name, namespace=None): + """Resolves the Avro name according to this name tracker's state. + + Args: + name: Name to resolve (absolute or relative). + namespace: Optional explicit namespace. + Returns: + The specified name, resolved according to this tracker. + """ + if namespace is None: namespace = self._default_namespace + return Name(name=name, namespace=namespace) + + def has_name(self, name, namespace=None): + avro_name = self.GetName(name=name, namespace=namespace) + return avro_name.fullname in self._names + + def get_name(self, name, namespace=None): + avro_name = self.GetName(name=name, namespace=namespace) + return self._names.get(avro_name.fullname, None) + + def GetSchema(self, name, namespace=None): + """Resolves an Avro schema by name. + + Args: + name: Name (relative or absolute) of the Avro schema to look up. + namespace: Optional explicit namespace. + Returns: + The schema with the specified name, if any, or None. + """ + avro_name = self.GetName(name=name, namespace=namespace) + return self._names.get(avro_name.fullname, None) + + def prune_namespace(self, properties): + """given a properties, return properties with namespace removed if + it matches the own default namespace + """ + if self.default_namespace is None: + # I have no default -- no change + return properties + if 'namespace' not in properties: + # he has no namespace - no change + return properties + if properties['namespace'] != self.default_namespace: + # we're different - leave his stuff alone + return properties + # we each have a namespace and it's redundant. delete his. + prunable = properties.copy() + del(prunable['namespace']) + return prunable + + def Register(self, schema): + """Registers a new named schema in this tracker. + + Args: + schema: Named Avro schema to register in this tracker. + """ + if schema.fullname in VALID_TYPES: + raise SchemaParseException( + '%s is a reserved type name.' % schema.fullname) + if schema.fullname in self.names: + raise SchemaParseException( + 'Avro name %r already exists.' % schema.fullname) + + logger.log(DEBUG_VERBOSE, 'Register new name for %r', schema.fullname) + self._names[schema.fullname] = schema + + +# ------------------------------------------------------------------------------ + + +class NamedSchema(Schema): + """Abstract base class for named schemas. + + Named schemas are enumerated in NAMED_TYPES. + """ + + def __init__( + self, + type, + name, + namespace=None, + names=None, + other_props=None, + ): + """Initializes a new named schema object. + + Args: + type: Type of the named schema. + name: Name (absolute or relative) of the schema. + namespace: Optional explicit namespace if name is relative. + names: Tracker to resolve and register Avro names. + other_props: Optional map of additional properties of the schema. + """ + assert (type in NAMED_TYPES), ('Invalid named type: %r' % type) + self._avro_name = names.GetName(name=name, namespace=namespace) + + super(NamedSchema, self).__init__(type, other_props) + + names.Register(self) + + self._props['name'] = self.name + if self.namespace: + self._props['namespace'] = self.namespace + + @property + def avro_name(self): + """Returns: the Name object describing this schema's name.""" + return self._avro_name + + @property + def name(self): + return self._avro_name.simple_name + + @property + def namespace(self): + return self._avro_name.namespace + + @property + def fullname(self): + return self._avro_name.fullname + + def name_ref(self, names): + """Reports this schema name relative to the specified name tracker. + + Args: + names: Avro name tracker to relativise this schema name against. + Returns: + This schema name, relativised against the specified name tracker. + """ + if self.namespace == names.default_namespace: + return self.name + else: + return self.fullname + + +# ------------------------------------------------------------------------------ + + +_NO_DEFAULT = object() + + +class Field(object): + """Representation of the schema of a field in a record.""" + + def __init__( + self, + type, + name, + index, + has_default, + default=_NO_DEFAULT, + order=None, + names=None, + doc=None, + other_props=None + ): + """Initializes a new Field object. + + Args: + type: Avro schema of the field. + name: Name of the field. + index: 0-based position of the field. + has_default: + default: + order: + names: + doc: + other_props: + """ + if (not isinstance(name, str)) or (len(name) == 0): + raise SchemaParseException('Invalid record field name: %r.' % name) + if (order is not None) and (order not in VALID_FIELD_SORT_ORDERS): + raise SchemaParseException('Invalid record field order: %r.' % order) + + # All properties of this record field: + self._props = {} + + self._has_default = has_default + if other_props: + self._props.update(other_props) + + self._index = index + self._type = self._props['type'] = type + self._name = self._props['name'] = name + + # TODO: check to ensure default is valid + if has_default: + self._props['default'] = default + + if order is not None: + self._props['order'] = order + + if doc is not None: + self._props['doc'] = doc + + @property + def type(self): + """Returns: the schema of this field.""" + return self._type + + @property + def name(self): + """Returns: this field name.""" + return self._name + + @property + def index(self): + """Returns: the 0-based index of this field in the record.""" + return self._index + + @property + def default(self): + return self._props['default'] + + @property + def has_default(self): + return self._has_default + + @property + def order(self): + return self._props.get('order', None) + + @property + def doc(self): + return self._props.get('doc', None) + + @property + def props(self): + return self._props + + @property + def other_props(self): + return FilterKeysOut(items=self._props, keys=FIELD_RESERVED_PROPS) + + def __str__(self): + return json.dumps(self.to_json(), cls=MappingProxyEncoder) + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = self.props.copy() + to_dump['type'] = self.type.to_json(names) + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(str(self)) + return to_cmp == json.loads(str(that)) + + +# ------------------------------------------------------------------------------ +# Primitive Types + + +class PrimitiveSchema(Schema): + """Schema of a primitive Avro type. + + Valid primitive types are defined in PRIMITIVE_TYPES. + """ + + def __init__(self, type, other_props=None): + """Initializes a new schema object for the specified primitive type. + + Args: + type: Type of the schema to construct. Must be primitive. + """ + if type not in PRIMITIVE_TYPES: + raise AvroException('%r is not a valid primitive type.' % type) + super(PrimitiveSchema, self).__init__(type, other_props=other_props) + + @property + def name(self): + """Returns: the simple name of this schema.""" + # The name of a primitive type is the type itself. + return self.type + + @property + def fullname(self): + """Returns: the fully qualified name of this schema.""" + # The full name is the simple name for primitive schema. + return self.name + + def to_json(self, names=None): + if len(self.props) == 1: + return self.fullname + else: + return self.props + + def __eq__(self, that): + return self.props == that.props + + +# ------------------------------------------------------------------------------ +# Complex Types (non-recursive) + + +class FixedSchema(NamedSchema): + def __init__( + self, + name, + namespace, + size, + names=None, + other_props=None, + ): + # Ensure valid ctor args + if not isinstance(size, int): + fail_msg = 'Fixed Schema requires a valid integer for size property.' + raise AvroException(fail_msg) + + super(FixedSchema, self).__init__( + type=FIXED, + name=name, + namespace=namespace, + names=names, + other_props=other_props, + ) + self._props['size'] = size + + @property + def size(self): + """Returns: the size of this fixed schema, in bytes.""" + return self._props['size'] + + def to_json(self, names=None): + if names is None: + names = Names() + if self.fullname in names.names: + return self.name_ref(names) + else: + names.names[self.fullname] = self + return names.prune_namespace(self.props) + + def __eq__(self, that): + return self.props == that.props + + +# ------------------------------------------------------------------------------ + + +class EnumSchema(NamedSchema): + def __init__( + self, + name, + namespace, + symbols, + names=None, + doc=None, + other_props=None, + ): + """Initializes a new enumeration schema object. + + Args: + name: Simple name of this enumeration. + namespace: Optional namespace. + symbols: Ordered list of symbols defined in this enumeration. + names: + doc: + other_props: + """ + symbols = tuple(symbols) + symbol_set = frozenset(symbols) + if (len(symbol_set) != len(symbols) + or not all(map(lambda symbol: isinstance(symbol, str), symbols))): + raise AvroException( + 'Invalid symbols for enum schema: %r.' % (symbols,)) + + super(EnumSchema, self).__init__( + type=ENUM, + name=name, + namespace=namespace, + names=names, + other_props=other_props, + ) + + self._props['symbols'] = symbols + if doc is not None: + self._props['doc'] = doc + + @property + def symbols(self): + """Returns: the symbols defined in this enum.""" + return self._props['symbols'] + + def to_json(self, names=None): + if names is None: + names = Names() + if self.fullname in names.names: + return self.name_ref(names) + else: + names.names[self.fullname] = self + return names.prune_namespace(self.props) + + def __eq__(self, that): + return self.props == that.props + + +# ------------------------------------------------------------------------------ +# Complex Types (recursive) + + +class ArraySchema(Schema): + """Schema of an array.""" + + def __init__(self, items, other_props=None): + """Initializes a new array schema object. + + Args: + items: Avro schema of the array items. + other_props: + """ + super(ArraySchema, self).__init__( + type=ARRAY, + other_props=other_props, + ) + self._items_schema = items + self._props['items'] = items + + @property + def items(self): + """Returns: the schema of the items in this array.""" + return self._items_schema + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = self.props.copy() + item_schema = self.items + to_dump['items'] = item_schema.to_json(names) + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(str(self)) + return to_cmp == json.loads(str(that)) + + +# ------------------------------------------------------------------------------ + + +class MapSchema(Schema): + """Schema of a map.""" + + def __init__(self, values, other_props=None): + """Initializes a new map schema object. + + Args: + values: Avro schema of the map values. + other_props: + """ + super(MapSchema, self).__init__( + type=MAP, + other_props=other_props, + ) + self._values_schema = values + self._props['values'] = values + + @property + def values(self): + """Returns: the schema of the values in this map.""" + return self._values_schema + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = self.props.copy() + to_dump['values'] = self.values.to_json(names) + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(str(self)) + return to_cmp == json.loads(str(that)) + + +# ------------------------------------------------------------------------------ + + +class UnionSchema(Schema): + """Schema of a union.""" + + def __init__(self, schemas): + """Initializes a new union schema object. + + Args: + schemas: Ordered collection of schema branches in the union. + """ + super(UnionSchema, self).__init__(type=UNION) + self._schemas = tuple(schemas) + + # Validate the schema branches: + + # All named schema names are unique: + named_branches = tuple( + filter(lambda schema: schema.type in NAMED_TYPES, self._schemas)) + unique_names = frozenset(map(lambda schema: schema.fullname, named_branches)) + if len(unique_names) != len(named_branches): + raise AvroException( + 'Invalid union branches with duplicate schema name:%s' + % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) + + # Types are unique within unnamed schemas, and union is not allowed: + unnamed_branches = tuple( + filter(lambda schema: schema.type not in NAMED_TYPES, self._schemas)) + unique_types = frozenset(map(lambda schema: schema.type, unnamed_branches)) + if UNION in unique_types: + raise AvroException( + 'Invalid union branches contain other unions:%s' + % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) + if len(unique_types) != len(unnamed_branches): + raise AvroException( + 'Invalid union branches with duplicate type:%s' + % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) + + @property + def schemas(self): + """Returns: the ordered list of schema branches in the union.""" + return self._schemas + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = [] + for schema in self.schemas: + to_dump.append(schema.to_json(names)) + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(str(self)) + return to_cmp == json.loads(str(that)) + + +# ------------------------------------------------------------------------------ + + +class ErrorUnionSchema(UnionSchema): + """Schema representing the declared errors of a protocol message.""" + + def __init__(self, schemas): + """Initializes an error-union schema. + + Args: + schema: collection of error schema. + """ + # TODO: check that string isn't already listed explicitly as an error. + # Prepend "string" to handle system errors + schemas = [PrimitiveSchema(type=STRING)] + list(schemas) + super(ErrorUnionSchema, self).__init__(schemas=schemas) + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = [] + for schema in self.schemas: + # Don't print the system error schema + if schema.type == STRING: continue + to_dump.append(schema.to_json(names)) + return to_dump + + +# ------------------------------------------------------------------------------ + + +class RecordSchema(NamedSchema): + """Schema of a record.""" + + @staticmethod + def _MakeField(index, field_desc, names): + """Builds field schemas from a list of field JSON descriptors. + + Args: + index: 0-based index of the field in the record. + field_desc: JSON descriptors of a record field. + names: Avro schema tracker. + Return: + The field schema. + """ + field_schema = SchemaFromJSONData( + json_data=field_desc['type'], + names=names, + ) + other_props = ( + dict(FilterKeysOut(items=field_desc, keys=FIELD_RESERVED_PROPS))) + return Field( + type=field_schema, + name=field_desc['name'], + index=index, + has_default=('default' in field_desc), + default=field_desc.get('default', _NO_DEFAULT), + order=field_desc.get('order', None), + names=names, + doc=field_desc.get('doc', None), + other_props=other_props, + ) + + @staticmethod + def _MakeFieldList(field_desc_list, names): + """Builds field schemas from a list of field JSON descriptors. + + Guarantees field name unicity. + + Args: + field_desc_list: collection of field JSON descriptors. + names: Avro schema tracker. + Yields + Field schemas. + """ + for index, field_desc in enumerate(field_desc_list): + yield RecordSchema._MakeField(index, field_desc, names) + + @staticmethod + def _MakeFieldMap(fields): + """Builds the field map. + + Guarantees field name unicity. + + Args: + fields: iterable of field schema. + Returns: + A read-only map of field schemas, indexed by name. + """ + field_map = {} + for field in fields: + if field.name in field_map: + raise SchemaParseException( + 'Duplicate record field name %r.' % field.name) + field_map[field.name] = field + return MappingProxyType(field_map) + + def __init__( + self, + name, + namespace, + fields=None, + make_fields=None, + names=None, + record_type=RECORD, + doc=None, + other_props=None + ): + """Initializes a new record schema object. + + Args: + name: Name of the record (absolute or relative). + namespace: Optional namespace the record belongs to, if name is relative. + fields: collection of fields to add to this record. + Exactly one of fields or make_fields must be specified. + make_fields: function creating the fields that belong to the record. + The function signature is: make_fields(names) -> ordered field list. + Exactly one of fields or make_fields must be specified. + names: + record_type: Type of the record: one of RECORD, ERROR or REQUEST. + Protocol requests are not named. + doc: + other_props: + """ + if record_type == REQUEST: + # Protocol requests are not named: + super(NamedSchema, self).__init__( + type=REQUEST, + other_props=other_props, + ) + elif record_type in [RECORD, ERROR]: + # Register this record name in the tracker: + super(RecordSchema, self).__init__( + type=record_type, + name=name, + namespace=namespace, + names=names, + other_props=other_props, + ) + else: + raise SchemaParseException( + 'Invalid record type: %r.' % record_type) + + if record_type in [RECORD, ERROR]: + avro_name = names.GetName(name=name, namespace=namespace) + nested_names = names.NewWithDefaultNamespace(namespace=avro_name.namespace) + elif record_type == REQUEST: + # Protocol request has no name: no need to change default namespace: + nested_names = names + + if fields is None: + fields = make_fields(names=nested_names) + else: + assert (make_fields is None) + self._fields = tuple(fields) + + self._field_map = RecordSchema._MakeFieldMap(self._fields) + + self._props['fields'] = fields + if doc is not None: + self._props['doc'] = doc + + @property + def fields(self): + """Returns: the field schemas, as an ordered tuple.""" + return self._fields + + @property + def field_map(self): + """Returns: a read-only map of the field schemas index by field names.""" + return self._field_map + + def to_json(self, names=None): + if names is None: + names = Names() + # Request records don't have names + if self.type == REQUEST: + return [f.to_json(names) for f in self.fields] + + if self.fullname in names.names: + return self.name_ref(names) + else: + names.names[self.fullname] = self + + to_dump = names.prune_namespace(self.props.copy()) + to_dump['fields'] = [f.to_json(names) for f in self.fields] + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(str(self)) + return to_cmp == json.loads(str(that)) + + +# ------------------------------------------------------------------------------ +# Module functions + + +def FilterKeysOut(items, keys): + """Filters a collection of (key, value) items. + + Exclude any item whose key belongs to keys. + + Args: + items: Dictionary of items to filter the keys out of. + keys: Keys to filter out. + Yields: + Filtered items. + """ + for key, value in items.items(): + if key in keys: continue + yield (key, value) + + +# ------------------------------------------------------------------------------ + + +def _SchemaFromJSONString(json_string, names): + if json_string in PRIMITIVE_TYPES: + return PrimitiveSchema(type=json_string) + else: + # Look for a known named schema: + schema = names.GetSchema(name=json_string) + if schema is None: + raise SchemaParseException( + 'Unknown named schema %r, known names: %r.' + % (json_string, sorted(names.names))) + return schema + + +def _SchemaFromJSONArray(json_array, names): + def MakeSchema(desc): + return SchemaFromJSONData(json_data=desc, names=names) + return UnionSchema(map(MakeSchema, json_array)) + + +def _SchemaFromJSONObject(json_object, names): + type = json_object.get('type') + if type is None: + raise SchemaParseException( + 'Avro schema JSON descriptor has no "type" property: %r' % json_object) + + other_props = dict( + FilterKeysOut(items=json_object, keys=SCHEMA_RESERVED_PROPS)) + + if type in PRIMITIVE_TYPES: + # FIXME should not ignore other properties + return PrimitiveSchema(type, other_props=other_props) + + elif type in NAMED_TYPES: + name = json_object.get('name') + namespace = json_object.get('namespace', names.default_namespace) + if type == FIXED: + size = json_object.get('size') + return FixedSchema(name, namespace, size, names, other_props) + elif type == ENUM: + symbols = json_object.get('symbols') + doc = json_object.get('doc') + return EnumSchema(name, namespace, symbols, names, doc, other_props) + + elif type in [RECORD, ERROR]: + field_desc_list = json_object.get('fields', ()) + + def MakeFields(names): + return tuple(RecordSchema._MakeFieldList(field_desc_list, names)) + + return RecordSchema( + name=name, + namespace=namespace, + make_fields=MakeFields, + names=names, + record_type=type, + doc=json_object.get('doc'), + other_props=other_props, + ) + else: + raise Exception('Internal error: unknown type %r.' % type) + + elif type in VALID_TYPES: + # Unnamed, non-primitive Avro type: + + if type == ARRAY: + items_desc = json_object.get('items') + if items_desc is None: + raise SchemaParseException( + 'Invalid array schema descriptor with no "items" : %r.' + % json_object) + return ArraySchema( + items=SchemaFromJSONData(items_desc, names), + other_props=other_props, + ) + + elif type == MAP: + values_desc = json_object.get('values') + if values_desc is None: + raise SchemaParseException( + 'Invalid map schema descriptor with no "values" : %r.' + % json_object) + return MapSchema( + values=SchemaFromJSONData(values_desc, names=names), + other_props=other_props, + ) + + elif type == ERROR_UNION: + error_desc_list = json_object.get('declared_errors') + assert (error_desc_list is not None) + error_schemas = map( + lambda desc: SchemaFromJSONData(desc, names=names), + error_desc_list) + return ErrorUnionSchema(schemas=error_schemas) + + else: + raise Exception('Internal error: unknown type %r.' % type) + + raise SchemaParseException( + 'Invalid JSON descriptor for an Avro schema: %r' % json_object) + + +# Parsers for the JSON data types: +_JSONDataParserTypeMap = { + str: _SchemaFromJSONString, + list: _SchemaFromJSONArray, + dict: _SchemaFromJSONObject, +} + + +def SchemaFromJSONData(json_data, names=None): + """Builds an Avro Schema from its JSON descriptor. + + Args: + json_data: JSON data representing the descriptor of the Avro schema. + names: Optional tracker for Avro named schemas. + Returns: + The Avro schema parsed from the JSON descriptor. + Raises: + SchemaParseException: if the descriptor is invalid. + """ + if names is None: + names = Names() + + # Select the appropriate parser based on the JSON data type: + parser = _JSONDataParserTypeMap.get(type(json_data)) + if parser is None: + raise SchemaParseException( + 'Invalid JSON descriptor for an Avro schema: %r.' % json_data) + return parser(json_data, names=names) + + +# ------------------------------------------------------------------------------ + + +def parse(json_string): + """Constructs a Schema from its JSON descriptor in text form. + + Args: + json_string: String representation of the JSON descriptor of the schema. + Returns: + The parsed schema. + Raises: + SchemaParseException: on JSON parsing error, + or if the JSON descriptor is invalid. + """ + try: + json_data = json.loads(json_string) + except Exception as exn: + raise SchemaParseException( + 'Error parsing schema from JSON: %r. ' + 'Error message: %r.' + % (json_string, exn)) + + # Initialize the names object + names = Names() + + # construct the Avro Schema object + return SchemaFromJSONData(json_data, names) + + +def Parse(json_string): + """Deprecated implementation of parse.""" + warnings.warn("`Parse` is deprecated in avro 1.9.2. " + "Please use `parse` (lowercase) instead.", + DeprecationWarning) + return parse(json_string) diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/__init__.py b/sdk/storage/azure-storage-blob/tests/_shared/avro/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/changeFeed.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/changeFeed.avro new file mode 100644 index 0000000000000000000000000000000000000000..67679fbbdb1920d5aa18a350794bb85a773f527f GIT binary patch literal 2514 zcmb7FTW{Mo6n28ZU9oO`DTcl5MNb=GMby=j-%=-A3ba_TG{dl>z$o&_VJnLoNf`+O z1H+#71NO9^wqG++l4T{XW6v-Q(&xhSedqRL`r+W{1zre#OgIJdSyAQrSM^Ath7rLf&(!gy;sVb=%QFT! zmDc3l-O5Mv$j=}oDqku5x8f{N<=KQM_bittypZ_mwTHr!A}W0}vBj z>w1Si7$|QI;FU#(ABt+O|83FNn2q9D^S8X6r#v^{tgAwnK$u2LJ1uQk^CbW)FeIAW zDde!eT?k%+Pz>Z%NqbtC1uo?^SJTdR$_piUZoThO^yycZJt`V#Oy_O|&Duc*GlA~M zn+_Ov${l4pO6IgN+AIrPvQ20A`9CSyqs(i$c5zWDLv(i{A73+Rs$t5@+g8$Kp}kx} zZSV#ZZXvZ~*O}CYR|T!Jc3*#;JFT)xSgo=xM<%}Imi#rcuHs3~bW#2UzgHmDtNn8# z{7 z(xbCY=p2_?xWKtCBt4?rD3*Pk#-c~QY-i7g&z12bwyCCR-52Yb)&#R7xGv_0BczS~ zQ_F-@LNtr0#k+&%6=@x~6{vn&TFdv#M;U6i+bP4df=k7S?A&!2w(#D2=1#w1zJo8G zJb!j{^lWhO^x(7=O687@XueX)9~*@NAERB1S^%^o6<>f5T>5*+u%WF2X2(u zag1U+NFYTth7=>}2#~xI1bz^_Dy8$wv>q!?Dfog>y@8#LmF>s|+zBDPDwK32;DXdE zKLemrPYcK#ljlni3r0XX?aFpcqhnSWw@Oc~T1nf}>{#8X!`MS^V!N)5+_&y{68aM~ zJ`P%+lswbqs&ckBlDL>VE2 z!C*K{89k>y3jGYbHbyAYKmt$y)Ihv6O&hbJ)QTa-SrVt2ow+o&DIG_)N2zBgX@p1= dC!rtHp`HrXw_$?fk?RMe00qGaeYa6C{{g9qEgk>> literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_0.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_0.avro new file mode 100644 index 0000000000000000000000000000000000000000..0f7fe1250eaf9ecb2613d54d72bcff8c24d1c6f0 GIT binary patch literal 83 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<2_DCL#r@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<0HDJA9S=cFd)DKT&sOk~o0 aZ0$NJmT#e)xa=Ddf#j4HMg|5)3>g3p?;JP) literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_10.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_10.avro new file mode 100644 index 0000000000000000000000000000000000000000..73f1bbec8e13650caf2ab91a9732dab0116338e2 GIT binary patch literal 119 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<2NR4bKK7Nja!DJUft6(v?G z=_n{=mZatugG6%j^U{@S8CLO#a!+&heyp%9z$<2HGLMKl CktUu1 literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_11.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_11.avro new file mode 100644 index 0000000000000000000000000000000000000000..ec48cd5280afb51562be9ab1ab659a8146c4e3b3 GIT binary patch literal 124 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<0%RV$TL7Nja!DJbP87AWZ` zD3vAVl%^Jg1ak88(v@l%xG(M~|6iaNU&7%1{<J91*qTWUrJYucTzJw8VtPw!#dX J@_5h{0RUv8Ejj=I literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_12.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_12.avro new file mode 100644 index 0000000000000000000000000000000000000000..11abb089d3d32661a69d20064fc4baa327c11587 GIT binary patch literal 106 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<1?L@O1S6lLb6E9odG<(1~- ofasijAU~Gj8_SDTx!XOZ4!^8isr#erx`;q>QWGNsLj{Hm0MPg;Z2$lO literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_13.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_13.avro new file mode 100644 index 0000000000000000000000000000000000000000..4436b93f1f5b76f9a75275b322dec01a15ef8c03 GIT binary patch literal 158 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<2hVyspwsVqoUvQkhgN=?o$ zN>S2LP|8cp1&f5F7MFndX_=`xDaAlF(FoN!`FZKcYSWZzV`~{+iiGlSx~415@Rski U;C#cEA_9*eH8C@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<2hW3E;zsVqoUvQkhgN=?o$ zN>S2LP|8cp1&jD(78ii{X_=`xDaAlF(Qwt#N_nL@IUvd6lA_GKbRY{Zhs7l4{Jdfu zCMm&ef!UgzSfo^oY#fGy6rh6G+E^r)mL=wtrh)_+#1*q;WtBIIdo$+M&YR+`B%+X< V_~?X(uTNsaLLpv;mIMKG6#!Y2TAly^ literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_2.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_2.avro new file mode 100644 index 0000000000000000000000000000000000000000..737f6a22740bb083d508407cb8eb26a0339ff5aa GIT binary patch literal 108 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<0HC>56!W#*+TF+AC$ov=c9 wp@Y&<{)P9MTslN7E_?cD`g!a4c^YrnVz7Z_Q)#ZFlGR~v35Ij)ebH3|04otFKL7v# literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_3.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_3.avro new file mode 100644 index 0000000000000000000000000000000000000000..47b4de9b7d64fbf31c9f1bdff3a68cae8228da48 GIT binary patch literal 97 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<2_DJ4~wq!ueNgk~<;TG*v~ k*D-eYkCR1drXniBMkYpPhQ2;teu9f6xf$3`OQEX)0D4~|rT_o{ literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_4.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_4.avro new file mode 100644 index 0000000000000000000000000000000000000000..8559ed0ef66ef48c96d953788a024ebc630d5cb0 GIT binary patch literal 87 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<1aDP`uBC^0Y!h>HkbC@A=6 Z!kroSJ;78&BwA??3o{dg?+@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<2_DCOkmr7JO@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<2_DW&D)CzdENcwa4iW-5C8 e{pOR))OpjdbcskL7tDBi$bp58;eG)(x-0+$?IhO# literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_7.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_7.avro new file mode 100644 index 0000000000000000000000000000000000000000..5a28fa77dc87fb422b9467d279ccdb49c7f54f90 GIT binary patch literal 94 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<0HD5d0=Cgr3mF_c*QP7upo hdYiM*UP<-OFEA&VH79Og!s2LSQUA)WvL literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_8.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_8.avro new file mode 100644 index 0000000000000000000000000000000000000000..4a1bbe3bd41ea785bac5b7466c986c16478285d2 GIT binary patch literal 124 zcmeZI%3@>@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<0@R4bKK7Nja!DJZ37R-~pV z=_n}WCFX(!LQ;!MK>Xs&DiGhWmf^*Q@Nh~YM*GtY%NloTUNlnX1EJ+mu3l%44q~<2(RV$TL7Nja!DJZ4pmF6nx zC@AG6=7RYlsl_EAesN`PQhrV`P)W3sBS^qWDYllO_SwP>M)mz)@`DdNTQWD*Ktv!p Msfm$+p#nn&0QrS5$p8QV literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_0.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_0.avro new file mode 100644 index 0000000000000000000000000000000000000000..91c2b2469e5432eb9ec2390151bc9ff3e90ceaa0 GIT binary patch literal 75 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCd6YmRN(`NUQtMNu2+Y1`d^mr~ K=3Z$L3=seW)g9yj literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_1.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_1.avro new file mode 100644 index 0000000000000000000000000000000000000000..01371934eba3764a31d53dd3f9bc1e461702d1ab GIT binary patch literal 88 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCg_M%=^K()Y^OP9s8P`rr>EQjK TbNPM6yO#l9L_`?j09_6MP7xss literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_10.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_10.avro new file mode 100644 index 0000000000000000000000000000000000000000..97aaaa0bb91a78930168294a4d9b4a235d6dd92a GIT binary patch literal 153 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCU8@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCovM{eDhpDTtQ3@T6AP4d6qL#m zb4pW-K>|7XdFe{E49};o`JAs(f6}V2H`n@m#e0!EjBHGaOiW2^Ovx+^bYP-8005xm BHPQe8 literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_12.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_12.avro new file mode 100644 index 0000000000000000000000000000000000000000..ddf42625f4f320290e6da4136c343b800d139419 GIT binary patch literal 105 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCO`?^GONuh{(v@@+lt7XoIwv2< fk7c+TwR{ENa=xFFChYuieom>MhzJuLpvwUOZ!acI literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_13.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_13.avro new file mode 100644 index 0000000000000000000000000000000000000000..277376ae1aa5801191c8824813be8f020324fbae GIT binary patch literal 157 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCXE9bQl~fj_Dp@Hg6{RNU7o{la zC@AG6=7L2+Qj1GK{Itx}oRngqnrMXTocz3WWVLBZwXwAfaYu7sT72Vj$vnXPB|UzL Mf`|wg9H7eq0E1I9(f|Me literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_14.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_14.avro new file mode 100644 index 0000000000000000000000000000000000000000..3c34ec843837039174125c7b6d7b86554bd35374 GIT binary patch literal 358 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC=P_3+l~fj_Dp@Hg6{RNU7o{la zC@AG6=7L3hGK&j9{Itx}oRngqnrOJ{XeE$QAj#sAqRhN>APX*s#U$taykZ{B$gLMxCiYvk+gsN+Yg$~W$O1+dCXS5M M#1sZ<(dbqH03+gO_W%F@ literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_2.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_2.avro new file mode 100644 index 0000000000000000000000000000000000000000..bf119d9e16f55f99c06d203079cd7b35812f2744 GIT binary patch literal 308 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC1(b?QiZb)kl^8Z(S$pN@?6xH= opWCzEF8(bga)wzaF{L;yu{b5oz|z9N63EuI1&ItZVRVlJ04jEH;{X5v literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_3.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_3.avro new file mode 100644 index 0000000000000000000000000000000000000000..d542117f7f6e950c5858d2f5242d03db1142d117 GIT binary patch literal 177 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC`IM3>OHzxK7|d0}Qa-liyMB4a cZm;{t@4v_iMj=BZV-wTFq~sLZvCypr0FtpdUjP6A literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_4.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_4.avro new file mode 100644 index 0000000000000000000000000000000000000000..b514fd8218419e4dd2b5dfe9bda9f4b678a1581f GIT binary patch literal 94 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCxs)>VN|YGx{5Z8JF?q=kk$qY} QI;waJMKqQOV?uOQ0FbsOYybcN literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_5.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_5.avro new file mode 100644 index 0000000000000000000000000000000000000000..29e8ca4d5f3559887e1628238dff6d8499f315a1 GIT binary patch literal 95 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCd6aVU^U{?VDi@v#HHgi6KXqZr S;X}KQ-W1VTB8&;qRRI97RwwrW literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_6.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_6.avro new file mode 100644 index 0000000000000000000000000000000000000000..df22b0f901a3004e75a9a922eeaf6c61b942fa5f GIT binary patch literal 116 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC`IORf@)Jvx81%y!RhK2p%#Hf7 U#^f4vcdkePgTpKrVlcW+0GJvkApigX literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_7.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_7.avro new file mode 100644 index 0000000000000000000000000000000000000000..1168f99d0d1977ba6a446b5b4599a96efd6f072f GIT binary patch literal 158 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC1(Z_qOOtX^l^6>5J}b~GkAB(O YA*We={AY~F0!9W9@R;mCEgIbl0PIvMivR!s literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_8.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_8.avro new file mode 100644 index 0000000000000000000000000000000000000000..b4136af69b603bc2695bcdffdcd69e9abb650888 GIT binary patch literal 123 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OCBdV23DhpDTtQ3^eGAmM3lynr7 u@)C2w0wJlzB_MurW)+BUSj#Y1s# literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_9.avro b/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_9.avro new file mode 100644 index 0000000000000000000000000000000000000000..90abc062240449b5df26a9aeeebf602cb9f38d73 GIT binary patch literal 134 zcmeZI%3@>@Nh~YM*GtY%NloU+E6vFf1M`cMGg5OC^Qx6fDhpDTtQ3?|^Gb7-bQF~G z5_7@)kksN55Wl!GHz_}-7^oy#$q^*rq!e4rpyG4uSOkOUm+zatwOxp?(H9Y6f&+9p E09Ilw2LJ#7 literal 0 HcmV?d00001 diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py new file mode 100644 index 000000000000..06f6968c98e3 --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py @@ -0,0 +1,110 @@ + +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import unittest + +from azure.storage.blob._shared.avro.datafile import DataFileReader +from azure.storage.blob._shared.avro.io import DatumReader + +SCHEMAS_TO_VALIDATE = ( + ('"null"', None), + ('"boolean"', True), + ('"string"', 'adsfasdf09809dsf-=adsf'), + ('"bytes"', b'12345abcd'), + ('"int"', 1234), + ('"long"', 1234), + ('"float"', 1234.0), + ('"double"', 1234.0), + ('{"type": "fixed", "name": "Test", "size": 1}', b'B'), + ('{"type": "enum", "name": "Test", "symbols": ["A", "B"]}', 'B'), + ('{"type": "array", "items": "long"}', [1, 3, 2]), + ('{"type": "map", "values": "long"}', {'a': 1, 'b': 3, 'c': 2}), + ('["string", "null", "long"]', None), + + (""" + { + "type": "record", + "name": "Test", + "fields": [{"name": "f", "type": "long"}] + } + """, + {'f': 5}), + + (""" + { + "type": "record", + "name": "Lisp", + "fields": [{ + "name": "value", + "type": [ + "null", + "string", + { + "type": "record", + "name": "Cons", + "fields": [{"name": "car", "type": "Lisp"}, + {"name": "cdr", "type": "Lisp"}] + } + ] + }] + } + """, + {'value': {'car': {'value': 'head'}, 'cdr': {'value': None}}}), +) + +CODECS_TO_VALIDATE = ('null', 'deflate') + + +class AvroReaderTests(unittest.TestCase): + def test_reader(self): + correct = 0 + nitems = 10 + for iexample, (writer_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE): + for codec in CODECS_TO_VALIDATE: + file_path = './samples/test_' + codec + '_' + str(iexample) + '.avro' + with open(file_path, 'rb') as reader: + datum_reader = DatumReader() + with DataFileReader(reader, datum_reader) as dfr: + round_trip_data = list(dfr) + if ([datum] * nitems) == round_trip_data: + correct += 1 + self.assertEqual( + correct, + len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE)) + + def test_change_feed(self): + file_path = './samples/changeFeed.avro' + with open(file_path, 'rb') as reader: + datum_reader = DatumReader() + with DataFileReader(reader, datum_reader) as dfr: + data = list(dfr) + self.assertEqual(1, len(data)) + expectedRecord = { + 'data': { + 'api': 'PutBlob', + 'blobPropertiesUpdated': None, + 'blobType': 'BlockBlob', + 'clientRequestId': '75b6c460-fcd0-11e9-87e2-85def057dae9', + 'contentLength': 12, + 'contentType': 'text/plain', + 'etag': '0x8D75EF45A3B8617', + 'previousInfo': None, + 'requestId': 'bb219c8e-401e-0028-1fdd-90f393000000', + 'sequencer': '000000000000000000000000000017140000000000000fcc', + 'snapshot': None, + 'storageDiagnostics': {'bid': 'd3053fa1-a006-0042-00dd-902bbb000000', + 'seq': '(5908,134,4044,0)', + 'sid': '5aaf98bf-f1d8-dd76-2dd2-9b60c689538d'}, + 'url': ''}, + 'eventTime': '2019-11-01T17:53:07.5106080Z', + 'eventType': 'BlobCreated', + 'id': 'bb219c8e-401e-0028-1fdd-90f393069ae4', + 'schemaVersion': 3, + 'subject': '/blobServices/default/containers/test/blobs/sdf.txt', + 'topic': '/subscriptions/ba45b233-e2ef-4169-8808-49eb0d8eba0d/resourceGroups/XClient/providers/Microsoft.Storage/storageAccounts/seanchangefeedstage'} + self.assertEqual(expectedRecord, data[0]) \ No newline at end of file From 32828fab4012b5ae04e344aa6ffba7ac1975b7da Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Fri, 10 Apr 2020 10:46:07 -0700 Subject: [PATCH 02/13] try fixing test... --- .../tests/_shared/avro/test_avro.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py index 06f6968c98e3..9f3c0dc18634 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py +++ b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py @@ -6,6 +6,8 @@ # license information. # -------------------------------------------------------------------------- +import inspect +import os import unittest from azure.storage.blob._shared.avro.datafile import DataFileReader @@ -61,12 +63,17 @@ class AvroReaderTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + test_file_path = inspect.getfile(cls) + cls._samples_dir_root = os.path.join(os.path.dirname(test_file_path), 'samples') + def test_reader(self): correct = 0 nitems = 10 for iexample, (writer_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE): for codec in CODECS_TO_VALIDATE: - file_path = './samples/test_' + codec + '_' + str(iexample) + '.avro' + file_path = os.path.join(AvroReaderTests._samples_dir_root, 'test_' + codec + '_' + str(iexample) + '.avro') with open(file_path, 'rb') as reader: datum_reader = DatumReader() with DataFileReader(reader, datum_reader) as dfr: @@ -78,7 +85,7 @@ def test_reader(self): len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE)) def test_change_feed(self): - file_path = './samples/changeFeed.avro' + file_path = os.path.join(AvroReaderTests._samples_dir_root, 'changeFeed.avro') with open(file_path, 'rb') as reader: datum_reader = DatumReader() with DataFileReader(reader, datum_reader) as dfr: From 06f6cbaafc6bb0cb8f0071931ee5c8c5a06febe0 Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Fri, 10 Apr 2020 23:07:13 -0700 Subject: [PATCH 03/13] falling in love with python compatibility... --- .../blob/_shared/avro/{io.py => avro_io.py} | 17 +++++++---- .../storage/blob/_shared/avro/datafile.py | 26 ++++++++-------- .../azure/storage/blob/_shared/avro/schema.py | 30 ++++++++----------- .../tests/_shared/avro/test_avro.py | 4 +-- 4 files changed, 40 insertions(+), 37 deletions(-) rename sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/{io.py => avro_io.py} (98%) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py similarity index 98% rename from sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/io.py rename to sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py index 48e795386f77..1707dc4e5a0e 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/io.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py @@ -45,9 +45,13 @@ import json import logging import struct +import sys from ..avro import schema + +PY3 = sys.version_info[0] == 3 + logger = logging.getLogger(__name__) # ------------------------------------------------------------------------------ @@ -220,11 +224,14 @@ def read_utf8(self): that many bytes of UTF-8 encoded character data. """ input_bytes = self.read_bytes() - try: - return input_bytes.decode('utf-8') - except UnicodeDecodeError as exn: - logger.error('Invalid UTF-8 input bytes: %r', input_bytes) - raise exn + if PY3: + try: + return input_bytes.decode('utf-8') + except UnicodeDecodeError as exn: + logger.error('Invalid UTF-8 input bytes: %r', input_bytes) + raise exn + else: + return unicode(input_bytes, "utf-8") def check_crc32(self, bytes): checksum = STRUCT_CRC32.unpack(self.read(4))[0]; diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py index 2da8c896c2f8..159607e06120 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py @@ -22,11 +22,13 @@ import io import logging +import sys import zlib -from ..avro import io as avro_io +from ..avro import avro_io from ..avro import schema +PY3 = sys.version_info[0] == 3 logger = logging.getLogger(__name__) @@ -36,11 +38,12 @@ # Version of the container file: VERSION = 1 -# Magic code that starts a data container file: -MAGIC = b'Obj' + bytes([VERSION]) - -# Size of the magic code, in number of bytes: -MAGIC_SIZE = len(MAGIC) +if PY3: + MAGIC = b'Obj' + bytes([VERSION]) + MAGIC_SIZE = len(MAGIC) +else: + MAGIC = 'Obj' + chr(VERSION) + MAGIC_SIZE = len(MAGIC) # Size of the synchronization marker, in number of bytes: SYNC_SIZE = 16 @@ -253,13 +256,6 @@ def _read_block_header(self): # "raw" (no zlib headers) decompression. See zlib.h. uncompressed = zlib.decompress(data, -15) self._datum_decoder = avro_io.BinaryDecoder(io.BytesIO(uncompressed)) - elif self.codec == 'snappy': - # Compressed data includes a 4-byte CRC32 checksum - length = self.raw_decoder.read_long() - data = self.raw_decoder.read(length - 4) - uncompressed = snappy.decompress(data) - self._datum_decoder = avro_io.BinaryDecoder(io.BytesIO(uncompressed)) - self.raw_decoder.check_crc32(uncompressed); else: raise DataFileException("Unknown codec: %r" % self.codec) @@ -292,6 +288,10 @@ def __next__(self): self._block_count -= 1 return datum + # PY2 + def next(self): + return self.__next__() + def close(self): """Close this reader.""" self.reader.close() diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py index 3f2260e85b49..65dfdc7a78a2 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py @@ -42,8 +42,14 @@ import json import logging import re +import sys import warnings -from types import MappingProxyType +from six import with_metaclass + +PY2 = sys.version_info[0] == 2 + +if PY2: + str = unicode logger = logging.getLogger(__name__) @@ -149,18 +155,8 @@ class SchemaParseException(AvroException): """Error while parsing a JSON schema descriptor.""" pass -# ------------------------------------------------------------------------------ -# Utilities -class MappingProxyEncoder(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, MappingProxyType): - return obj.copy() - return json.JSONEncoder.default(self, obj) - -# ------------------------------------------------------------------------------ - -class Schema(object, metaclass=abc.ABCMeta): +class Schema(with_metaclass(abc.ABCMeta, object)): """Abstract base class for all Schema classes.""" def __init__(self, type, other_props=None): @@ -205,9 +201,9 @@ def props(self): JSON properties of this schema are directly generated from this dict. Returns: - A read-only dictionary of properties associated to this schema. + A dictionary of properties associated to this schema. """ - return MappingProxyType(self._props) + return self._props @property def other_props(self): @@ -216,7 +212,7 @@ def other_props(self): def __str__(self): """Returns: the JSON representation of this schema.""" - return json.dumps(self.to_json(), cls=MappingProxyEncoder) + return json.dumps(self.to_json()) @abc.abstractmethod def to_json(self, names): @@ -955,7 +951,7 @@ def _MakeFieldMap(fields): Args: fields: iterable of field schema. Returns: - A read-only map of field schemas, indexed by name. + A map of field schemas, indexed by name. """ field_map = {} for field in fields: @@ -963,7 +959,7 @@ def _MakeFieldMap(fields): raise SchemaParseException( 'Duplicate record field name %r.' % field.name) field_map[field.name] = field - return MappingProxyType(field_map) + return field_map def __init__( self, diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py index 9f3c0dc18634..980a6bc651ea 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py +++ b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py @@ -9,9 +9,9 @@ import inspect import os import unittest - +from io import open from azure.storage.blob._shared.avro.datafile import DataFileReader -from azure.storage.blob._shared.avro.io import DatumReader +from azure.storage.blob._shared.avro.avro_io import DatumReader SCHEMAS_TO_VALIDATE = ( ('"null"', None), From dcec5c8f32f2ffdb4b560b5b961036e58e1ac330 Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Tue, 14 Apr 2020 18:16:45 -0700 Subject: [PATCH 04/13] make linter happy. --- .../storage/blob/_shared/avro/__init__.py | 5 + .../storage/blob/_shared/avro/avro_io.py | 1162 +++++----- .../storage/blob/_shared/avro/datafile.py | 425 ++-- .../azure/storage/blob/_shared/avro/schema.py | 2031 ++++++++--------- 4 files changed, 1799 insertions(+), 1824 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/__init__.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/__init__.py index e69de29bb2d1..5b396cd202e8 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/__init__.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/__init__.py @@ -0,0 +1,5 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py index 1707dc4e5a0e..e488dc5b6d15 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py @@ -1,22 +1,8 @@ -#!/usr/bin/env python3 -# -*- mode: python -*- -# -*- coding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- """Input/output utilities. @@ -49,7 +35,6 @@ from ..avro import schema - PY3 = sys.version_info[0] == 3 logger = logging.getLogger(__name__) @@ -63,10 +48,10 @@ LONG_MIN_VALUE = -(1 << 63) LONG_MAX_VALUE = (1 << 63) - 1 -STRUCT_INT = struct.Struct('!I') # big-endian unsigned int -STRUCT_FLOAT = struct.Struct('I') # big-endian unsigned int +STRUCT_CRC32 = struct.Struct('>I') # big-endian unsigned int # ------------------------------------------------------------------------------ @@ -74,591 +59,616 @@ class AvroTypeException(schema.AvroException): - """Raised when datum is not an example of schema.""" - def __init__(self, expected_schema, datum): - pretty_expected = json.dumps(json.loads(str(expected_schema)), indent=2) - fail_msg = "The datum %s is not an example of the schema %s"\ - % (datum, pretty_expected) - schema.AvroException.__init__(self, fail_msg) + """Raised when datum is not an example of schema.""" + + def __init__(self, expected_schema, datum): + pretty_expected = json.dumps(json.loads(str(expected_schema)), indent=2) + fail_msg = "The datum %s is not an example of the schema %s" \ + % (datum, pretty_expected) + schema.AvroException.__init__(self, fail_msg) class SchemaResolutionException(schema.AvroException): - def __init__(self, fail_msg, writer_schema=None, reader_schema=None): - pretty_writers = json.dumps(json.loads(str(writer_schema)), indent=2) - pretty_readers = json.dumps(json.loads(str(reader_schema)), indent=2) - if writer_schema: fail_msg += "\nWriter's Schema: %s" % pretty_writers - if reader_schema: fail_msg += "\nReader's Schema: %s" % pretty_readers - schema.AvroException.__init__(self, fail_msg) + def __init__(self, fail_msg, writer_schema=None, reader_schema=None): + pretty_writers = json.dumps(json.loads(str(writer_schema)), indent=2) + pretty_readers = json.dumps(json.loads(str(reader_schema)), indent=2) + if writer_schema: + fail_msg += "\nWriter's Schema: %s" % pretty_writers + if reader_schema: + fail_msg += "\nReader's Schema: %s" % pretty_readers + schema.AvroException.__init__(self, fail_msg) # ------------------------------------------------------------------------------ # Validate _valid = { - 'null': lambda s, d: d is None, - 'boolean': lambda s, d: isinstance(d, bool), - 'string': lambda s, d: isinstance(d, str), - 'bytes': lambda s, d: isinstance(d, bytes), - 'int': lambda s, d: isinstance(d, int) and (INT_MIN_VALUE <= d <= INT_MAX_VALUE), - 'long': lambda s, d: isinstance(d, int) and (LONG_MIN_VALUE <= d <= LONG_MAX_VALUE), - 'float': lambda s, d: isinstance(d, (int, float)), - 'fixed': lambda s, d: isinstance(d, bytes) and len(d) == s.size, - 'enum': lambda s, d: d in s.symbols, - 'array': lambda s, d: isinstance(d, list) and all(Validate(s.items, item) for item in d), - 'map': lambda s, d: (isinstance(d, dict) and all(isinstance(key, str) for key in d) - and all(Validate(s.values, value) for value in d.values())), - 'union': lambda s, d: any(Validate(branch, d) for branch in s.schemas), - 'record': lambda s, d: (isinstance(d, dict) - and all(Validate(f.type, d.get(f.name)) for f in s.fields) - and {f.name for f in s.fields}.issuperset(d.keys())) + 'null': lambda s, d: d is None, + 'boolean': lambda s, d: isinstance(d, bool), + 'string': lambda s, d: isinstance(d, str), + 'bytes': lambda s, d: isinstance(d, bytes), + 'int': lambda s, d: isinstance(d, int) and (INT_MIN_VALUE <= d <= INT_MAX_VALUE), + 'long': lambda s, d: isinstance(d, int) and (LONG_MIN_VALUE <= d <= LONG_MAX_VALUE), + 'float': lambda s, d: isinstance(d, (int, float)), + 'fixed': lambda s, d: isinstance(d, bytes) and len(d) == s.size, + 'enum': lambda s, d: d in s.symbols, + 'array': lambda s, d: isinstance(d, list) and all(Validate(s.items, item) for item in d), + 'map': lambda s, d: (isinstance(d, dict) and all(isinstance(key, str) for key in d) + and all(Validate(s.values, value) for value in d.values())), + 'union': lambda s, d: any(Validate(branch, d) for branch in s.schemas), + 'record': lambda s, d: (isinstance(d, dict) + and all(Validate(f.type, d.get(f.name)) for f in s.fields) + and {f.name for f in s.fields}.issuperset(d.keys())) } _valid['double'] = _valid['float'] _valid['error_union'] = _valid['union'] _valid['error'] = _valid['request'] = _valid['record'] -def Validate(expected_schema, datum): - """Determines if a python datum is an instance of a schema. - - Args: - expected_schema: Schema to validate against. - datum: Datum to validate. - Returns: - True if the datum is an instance of the schema. - """ - try: - return _valid[expected_schema.type](expected_schema, datum) - except KeyError: - raise AvroTypeException(expected_schema, datum) - - -# ------------------------------------------------------------------------------ -# Decoder/Encoder - -class BinaryDecoder(object): - """Read leaf values.""" - def __init__(self, reader): - """ - reader is a Python object on which we can call read, seek, and tell. - """ - self._reader = reader - - @property - def reader(self): - """Reports the reader used by this decoder.""" - return self._reader - - def read(self, n): - """Read n bytes. +def Validate(expected_schema, datum): + """Determines if a python datum is an instance of a schema. Args: - n: Number of bytes to read. + expected_schema: Schema to validate against. + datum: Datum to validate. Returns: - The next n bytes from the input. - """ - assert (n >= 0), n - input_bytes = self.reader.read(n) - assert (len(input_bytes) == n), input_bytes - return input_bytes - - def read_null(self): - """ - null is written as zero bytes - """ - return None - - def read_boolean(self): - """ - a boolean is written as a single byte - whose value is either 0 (false) or 1 (true). - """ - return ord(self.read(1)) == 1 - - def read_int(self): - """ - int and long values are written using variable-length, zig-zag coding. - """ - return self.read_long() - - def read_long(self): - """ - int and long values are written using variable-length, zig-zag coding. - """ - b = ord(self.read(1)) - n = b & 0x7F - shift = 7 - while (b & 0x80) != 0: - b = ord(self.read(1)) - n |= (b & 0x7F) << shift - shift += 7 - datum = (n >> 1) ^ -(n & 1) - return datum - - def read_float(self): - """ - A float is written as 4 bytes. - The float is converted into a 32-bit integer using a method equivalent to - Java's floatToIntBits and then encoded in little-endian format. - """ - return STRUCT_FLOAT.unpack(self.read(4))[0] - - def read_double(self): - """ - A double is written as 8 bytes. - The double is converted into a 64-bit integer using a method equivalent to - Java's doubleToLongBits and then encoded in little-endian format. - """ - return STRUCT_DOUBLE.unpack(self.read(8))[0] - - def read_bytes(self): + True if the datum is an instance of the schema. """ - Bytes are encoded as a long followed by that many bytes of data. - """ - nbytes = self.read_long() - assert (nbytes >= 0), nbytes - return self.read(nbytes) - - def read_utf8(self): - """ - A string is encoded as a long followed by - that many bytes of UTF-8 encoded character data. - """ - input_bytes = self.read_bytes() - if PY3: - try: - return input_bytes.decode('utf-8') - except UnicodeDecodeError as exn: - logger.error('Invalid UTF-8 input bytes: %r', input_bytes) - raise exn - else: - return unicode(input_bytes, "utf-8") - - def check_crc32(self, bytes): - checksum = STRUCT_CRC32.unpack(self.read(4))[0]; - if binascii.crc32(bytes) & 0xffffffff != checksum: - raise schema.AvroException("Checksum failure") - - def skip_null(self): - pass - - def skip_boolean(self): - self.skip(1) - - def skip_int(self): - self.skip_long() - - def skip_long(self): - b = ord(self.read(1)) - while (b & 0x80) != 0: - b = ord(self.read(1)) + try: + return _valid[expected_schema.type](expected_schema, datum) + except KeyError: + raise AvroTypeException(expected_schema, datum) - def skip_float(self): - self.skip(4) - def skip_double(self): - self.skip(8) +# ------------------------------------------------------------------------------ +# Decoder/Encoder - def skip_bytes(self): - self.skip(self.read_long()) - def skip_utf8(self): - self.skip_bytes() +class BinaryDecoder(object): + """Read leaf values.""" + + def __init__(self, reader): + """ + reader is a Python object on which we can call read, seek, and tell. + """ + self._reader = reader + + @property + def reader(self): + """Reports the reader used by this decoder.""" + return self._reader + + def read(self, n): + """Read n bytes. + + Args: + n: Number of bytes to read. + Returns: + The next n bytes from the input. + """ + assert (n >= 0), n + input_bytes = self.reader.read(n) + assert (len(input_bytes) == n), input_bytes + return input_bytes + + @staticmethod + def read_null(): + """ + null is written as zero bytes + """ + return None + + def read_boolean(self): + """ + a boolean is written as a single byte + whose value is either 0 (false) or 1 (true). + """ + return ord(self.read(1)) == 1 + + def read_int(self): + """ + int and long values are written using variable-length, zig-zag coding. + """ + return self.read_long() + + def read_long(self): + """ + int and long values are written using variable-length, zig-zag coding. + """ + b = ord(self.read(1)) + n = b & 0x7F + shift = 7 + while (b & 0x80) != 0: + b = ord(self.read(1)) + n |= (b & 0x7F) << shift + shift += 7 + datum = (n >> 1) ^ -(n & 1) + return datum + + def read_float(self): + """ + A float is written as 4 bytes. + The float is converted into a 32-bit integer using a method equivalent to + Java's floatToIntBits and then encoded in little-endian format. + """ + return STRUCT_FLOAT.unpack(self.read(4))[0] + + def read_double(self): + """ + A double is written as 8 bytes. + The double is converted into a 64-bit integer using a method equivalent to + Java's doubleToLongBits and then encoded in little-endian format. + """ + return STRUCT_DOUBLE.unpack(self.read(8))[0] + + def read_bytes(self): + """ + Bytes are encoded as a long followed by that many bytes of data. + """ + nbytes = self.read_long() + assert (nbytes >= 0), nbytes + return self.read(nbytes) + + def read_utf8(self): + """ + A string is encoded as a long followed by + that many bytes of UTF-8 encoded character data. + """ + input_bytes = self.read_bytes() + if PY3: + try: + return input_bytes.decode('utf-8') + except UnicodeDecodeError as exn: + logger.error('Invalid UTF-8 input bytes: %r', input_bytes) + raise exn + else: + # PY2 + return unicode(input_bytes, "utf-8") # pylint: disable=undefined-variable + + def check_crc32(self, input_bytes): + checksum = STRUCT_CRC32.unpack(self.read(4))[0] + if binascii.crc32(input_bytes) & 0xffffffff != checksum: + raise schema.AvroException("Checksum failure") + + def skip_null(self): + pass + + def skip_boolean(self): + self.skip(1) + + def skip_int(self): + self.skip_long() + + def skip_long(self): + b = ord(self.read(1)) + while (b & 0x80) != 0: + b = ord(self.read(1)) + + def skip_float(self): + self.skip(4) + + def skip_double(self): + self.skip(8) + + def skip_bytes(self): + self.skip(self.read_long()) + + def skip_utf8(self): + self.skip_bytes() + + def skip(self, n): + self.reader.seek(self.reader.tell() + n) - def skip(self, n): - self.reader.seek(self.reader.tell() + n) # ------------------------------------------------------------------------------ # DatumReader/Writer class DatumReader(object): - """Deserialize Avro-encoded data into a Python data structure.""" - @staticmethod - def check_props(schema_one, schema_two, prop_list): - for prop in prop_list: - if getattr(schema_one, prop) != getattr(schema_two, prop): - return False - return True - - @staticmethod - def match_schemas(writer_schema, reader_schema): - w_type = writer_schema.type - r_type = reader_schema.type - if 'union' in [w_type, r_type] or 'error_union' in [w_type, r_type]: - return True - elif (w_type in schema.PRIMITIVE_TYPES and r_type in schema.PRIMITIVE_TYPES - and w_type == r_type): - return True - elif (w_type == r_type == 'record' and - DatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - return True - elif (w_type == r_type == 'error' and - DatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - return True - elif (w_type == r_type == 'request'): - return True - elif (w_type == r_type == 'fixed' and - DatumReader.check_props(writer_schema, reader_schema, - ['fullname', 'size'])): - return True - elif (w_type == r_type == 'enum' and - DatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - return True - elif (w_type == r_type == 'map' and - DatumReader.check_props(writer_schema.values, - reader_schema.values, ['type'])): - return True - elif (w_type == r_type == 'array' and - DatumReader.check_props(writer_schema.items, - reader_schema.items, ['type'])): - return True - - # Handle schema promotion - if w_type == 'int' and r_type in ['long', 'float', 'double']: - return True - elif w_type == 'long' and r_type in ['float', 'double']: - return True - elif w_type == 'float' and r_type == 'double': - return True - return False - - def __init__(self, writer_schema=None, reader_schema=None): - """ - As defined in the Avro specification, we call the schema encoded - in the data the "writer's schema", and the schema expected by the - reader the "reader's schema". - """ - self._writer_schema = writer_schema - self._reader_schema = reader_schema - - # read/write properties - def set_writer_schema(self, writer_schema): - self._writer_schema = writer_schema - writer_schema = property(lambda self: self._writer_schema, - set_writer_schema) - def set_reader_schema(self, reader_schema): - self._reader_schema = reader_schema - reader_schema = property(lambda self: self._reader_schema, - set_reader_schema) - - def read(self, decoder): - if self.reader_schema is None: - self.reader_schema = self.writer_schema - return self.read_data(self.writer_schema, self.reader_schema, decoder) - - def read_data(self, writer_schema, reader_schema, decoder): - # schema matching - if not DatumReader.match_schemas(writer_schema, reader_schema): - fail_msg = 'Schemas do not match.' - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - - # schema resolution: reader's schema is a union, writer's schema is not - if (writer_schema.type not in ['union', 'error_union'] - and reader_schema.type in ['union', 'error_union']): - for s in reader_schema.schemas: - if DatumReader.match_schemas(writer_schema, s): - return self.read_data(writer_schema, s, decoder) - fail_msg = 'Schemas do not match.' - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - - # function dispatch for reading data based on type of writer's schema - if writer_schema.type == 'null': - return decoder.read_null() - elif writer_schema.type == 'boolean': - return decoder.read_boolean() - elif writer_schema.type == 'string': - return decoder.read_utf8() - elif writer_schema.type == 'int': - return decoder.read_int() - elif writer_schema.type == 'long': - return decoder.read_long() - elif writer_schema.type == 'float': - return decoder.read_float() - elif writer_schema.type == 'double': - return decoder.read_double() - elif writer_schema.type == 'bytes': - return decoder.read_bytes() - elif writer_schema.type == 'fixed': - return self.read_fixed(writer_schema, reader_schema, decoder) - elif writer_schema.type == 'enum': - return self.read_enum(writer_schema, reader_schema, decoder) - elif writer_schema.type == 'array': - return self.read_array(writer_schema, reader_schema, decoder) - elif writer_schema.type == 'map': - return self.read_map(writer_schema, reader_schema, decoder) - elif writer_schema.type in ['union', 'error_union']: - return self.read_union(writer_schema, reader_schema, decoder) - elif writer_schema.type in ['record', 'error', 'request']: - return self.read_record(writer_schema, reader_schema, decoder) - else: - fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type - raise schema.AvroException(fail_msg) - - def skip_data(self, writer_schema, decoder): - if writer_schema.type == 'null': - return decoder.skip_null() - elif writer_schema.type == 'boolean': - return decoder.skip_boolean() - elif writer_schema.type == 'string': - return decoder.skip_utf8() - elif writer_schema.type == 'int': - return decoder.skip_int() - elif writer_schema.type == 'long': - return decoder.skip_long() - elif writer_schema.type == 'float': - return decoder.skip_float() - elif writer_schema.type == 'double': - return decoder.skip_double() - elif writer_schema.type == 'bytes': - return decoder.skip_bytes() - elif writer_schema.type == 'fixed': - return self.skip_fixed(writer_schema, decoder) - elif writer_schema.type == 'enum': - return self.skip_enum(writer_schema, decoder) - elif writer_schema.type == 'array': - return self.skip_array(writer_schema, decoder) - elif writer_schema.type == 'map': - return self.skip_map(writer_schema, decoder) - elif writer_schema.type in ['union', 'error_union']: - return self.skip_union(writer_schema, decoder) - elif writer_schema.type in ['record', 'error', 'request']: - return self.skip_record(writer_schema, decoder) - else: - fail_msg = "Unknown schema type: %s" % writer_schema.type - raise schema.AvroException(fail_msg) - - def read_fixed(self, writer_schema, reader_schema, decoder): - """ - Fixed instances are encoded using the number of bytes declared - in the schema. - """ - return decoder.read(writer_schema.size) - - def skip_fixed(self, writer_schema, decoder): - return decoder.skip(writer_schema.size) - - def read_enum(self, writer_schema, reader_schema, decoder): - """ - An enum is encoded by a int, representing the zero-based position - of the symbol in the schema. - """ - # read data - index_of_symbol = decoder.read_int() - if index_of_symbol >= len(writer_schema.symbols): - fail_msg = "Can't access enum index %d for enum with %d symbols"\ - % (index_of_symbol, len(writer_schema.symbols)) - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - read_symbol = writer_schema.symbols[index_of_symbol] - - # schema resolution - if read_symbol not in reader_schema.symbols: - fail_msg = "Symbol %s not present in Reader's Schema" % read_symbol - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - - return read_symbol - - def skip_enum(self, writer_schema, decoder): - return decoder.skip_int() - - def read_array(self, writer_schema, reader_schema, decoder): - """ - Arrays are encoded as a series of blocks. - - Each block consists of a long count value, - followed by that many array items. - A block with count zero indicates the end of the array. - Each item is encoded per the array's item schema. - - If a block's count is negative, - then the count is followed immediately by a long block size, - indicating the number of bytes in the block. - The actual count in this case - is the absolute value of the count written. - """ - read_items = [] - block_count = decoder.read_long() - while block_count != 0: - if block_count < 0: - block_count = -block_count - block_size = decoder.read_long() - for i in range(block_count): - read_items.append(self.read_data(writer_schema.items, - reader_schema.items, decoder)) - block_count = decoder.read_long() - return read_items - - def skip_array(self, writer_schema, decoder): - block_count = decoder.read_long() - while block_count != 0: - if block_count < 0: - block_size = decoder.read_long() - decoder.skip(block_size) - else: - for i in range(block_count): - self.skip_data(writer_schema.items, decoder) - block_count = decoder.read_long() - - def read_map(self, writer_schema, reader_schema, decoder): - """ - Maps are encoded as a series of blocks. - - Each block consists of a long count value, - followed by that many key/value pairs. - A block with count zero indicates the end of the map. - Each item is encoded per the map's value schema. - - If a block's count is negative, - then the count is followed immediately by a long block size, - indicating the number of bytes in the block. - The actual count in this case - is the absolute value of the count written. - """ - read_items = {} - block_count = decoder.read_long() - while block_count != 0: - if block_count < 0: - block_count = -block_count - block_size = decoder.read_long() - for i in range(block_count): - key = decoder.read_utf8() - read_items[key] = self.read_data(writer_schema.values, - reader_schema.values, decoder) - block_count = decoder.read_long() - return read_items - - def skip_map(self, writer_schema, decoder): - block_count = decoder.read_long() - while block_count != 0: - if block_count < 0: - block_size = decoder.read_long() - decoder.skip(block_size) - else: - for i in range(block_count): - decoder.skip_utf8() - self.skip_data(writer_schema.values, decoder) - block_count = decoder.read_long() - - def read_union(self, writer_schema, reader_schema, decoder): - """ - A union is encoded by first writing a long value indicating - the zero-based position within the union of the schema of its value. - The value is then encoded per the indicated schema within the union. - """ - # schema resolution - index_of_schema = int(decoder.read_long()) - if index_of_schema >= len(writer_schema.schemas): - fail_msg = "Can't access branch index %d for union with %d branches"\ - % (index_of_schema, len(writer_schema.schemas)) - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - selected_writer_schema = writer_schema.schemas[index_of_schema] - - # read data - return self.read_data(selected_writer_schema, reader_schema, decoder) - - def skip_union(self, writer_schema, decoder): - index_of_schema = int(decoder.read_long()) - if index_of_schema >= len(writer_schema.schemas): - fail_msg = "Can't access branch index %d for union with %d branches"\ - % (index_of_schema, len(writer_schema.schemas)) - raise SchemaResolutionException(fail_msg, writer_schema) - return self.skip_data(writer_schema.schemas[index_of_schema], decoder) - - def read_record(self, writer_schema, reader_schema, decoder): - """ - A record is encoded by encoding the values of its fields - in the order that they are declared. In other words, a record - is encoded as just the concatenation of the encodings of its fields. - Field values are encoded per their schema. - - Schema Resolution: - * the ordering of fields may be different: fields are matched by name. - * schemas for fields with the same name in both records are resolved - recursively. - * if the writer's record contains a field with a name not present in the - reader's record, the writer's value for that field is ignored. - * if the reader's record schema has a field that contains a default value, - and writer's schema does not have a field with the same name, then the - reader should use the default value from its field. - * if the reader's record schema has a field with no default value, and - writer's schema does not have a field with the same name, then the - field's value is unset. - """ - # schema resolution - readers_fields_dict = reader_schema.field_map - read_record = {} - for field in writer_schema.fields: - readers_field = readers_fields_dict.get(field.name) - if readers_field is not None: - field_val = self.read_data(field.type, readers_field.type, decoder) - read_record[field.name] = field_val - else: - self.skip_data(field.type, decoder) - - # fill in default values - if len(readers_fields_dict) > len(read_record): - writers_fields_dict = writer_schema.field_map - for field_name, field in readers_fields_dict.items(): - if field_name not in writers_fields_dict: - if field.has_default: - field_val = self._read_default_value(field.type, field.default) - read_record[field.name] = field_val - else: - fail_msg = 'No default value for field %s' % field_name - raise SchemaResolutionException(fail_msg, writer_schema, - reader_schema) - return read_record - - def skip_record(self, writer_schema, decoder): - for field in writer_schema.fields: - self.skip_data(field.type, decoder) - - def _read_default_value(self, field_schema, default_value): - """ - Basically a JSON Decoder? - """ - if field_schema.type == 'null': - return None - elif field_schema.type == 'boolean': - return bool(default_value) - elif field_schema.type == 'int': - return int(default_value) - elif field_schema.type == 'long': - return int(default_value) - elif field_schema.type in ['float', 'double']: - return float(default_value) - elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']: - return default_value - elif field_schema.type == 'array': - read_array = [] - for json_val in default_value: - item_val = self._read_default_value(field_schema.items, json_val) - read_array.append(item_val) - return read_array - elif field_schema.type == 'map': - read_map = {} - for key, json_val in default_value.items(): - map_val = self._read_default_value(field_schema.values, json_val) - read_map[key] = map_val - return read_map - elif field_schema.type in ['union', 'error_union']: - return self._read_default_value(field_schema.schemas[0], default_value) - elif field_schema.type == 'record': - read_record = {} - for field in field_schema.fields: - json_val = default_value.get(field.name) - if json_val is None: json_val = field.default - field_val = self._read_default_value(field.type, json_val) - read_record[field.name] = field_val - return read_record - else: - fail_msg = 'Unknown type: %s' % field_schema.type - raise schema.AvroException(fail_msg) + """Deserialize Avro-encoded data into a Python data structure.""" + + @staticmethod + def check_props(schema_one, schema_two, prop_list): + for prop in prop_list: + if getattr(schema_one, prop) != getattr(schema_two, prop): + return False + return True + + @staticmethod + def match_schemas(writer_schema, reader_schema): + w_type = writer_schema.type + r_type = reader_schema.type + if 'union' in [w_type, r_type] or 'error_union' in [w_type, r_type]: + result = True + elif (w_type in schema.PRIMITIVE_TYPES and r_type in schema.PRIMITIVE_TYPES + and w_type == r_type): + result = True + elif (w_type == r_type == 'record' and + DatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + result = True + elif (w_type == r_type == 'error' and + DatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + result = True + elif w_type == r_type == 'request': + return True + elif (w_type == r_type == 'fixed' and + DatumReader.check_props(writer_schema, reader_schema, + ['fullname', 'size'])): + result = True + elif (w_type == r_type == 'enum' and + DatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + result = True + elif (w_type == r_type == 'map' and + DatumReader.check_props(writer_schema.values, + reader_schema.values, ['type'])): + result = True + elif (w_type == r_type == 'array' and + DatumReader.check_props(writer_schema.items, + reader_schema.items, ['type'])): + result = True + + # Handle schema promotion + elif w_type == 'int' and r_type in ['long', 'float', 'double']: + result = True + elif w_type == 'long' and r_type in ['float', 'double']: + result = True + elif w_type == 'float' and r_type == 'double': + result = True + else: + result = False + return result + + def __init__(self, writer_schema=None, reader_schema=None): + """ + As defined in the Avro specification, we call the schema encoded + in the data the "writer's schema", and the schema expected by the + reader the "reader's schema". + """ + self._writer_schema = writer_schema + self._reader_schema = reader_schema + + # read/write properties + def set_writer_schema(self, writer_schema): + self._writer_schema = writer_schema + + writer_schema = property(lambda self: self._writer_schema, + set_writer_schema) + + def set_reader_schema(self, reader_schema): + self._reader_schema = reader_schema + + reader_schema = property(lambda self: self._reader_schema, + set_reader_schema) + + def read(self, decoder): + if self.reader_schema is None: + self.reader_schema = self.writer_schema + return self.read_data(self.writer_schema, self.reader_schema, decoder) + + def read_data(self, writer_schema, reader_schema, decoder): + # schema matching + if not DatumReader.match_schemas(writer_schema, reader_schema): + fail_msg = 'Schemas do not match.' + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + # schema resolution: reader's schema is a union, writer's schema is not + if (writer_schema.type not in ['union', 'error_union'] + and reader_schema.type in ['union', 'error_union']): + for s in reader_schema.schemas: + if DatumReader.match_schemas(writer_schema, s): + return self.read_data(writer_schema, s, decoder) + fail_msg = 'Schemas do not match.' + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + # function dispatch for reading data based on type of writer's schema + if writer_schema.type == 'null': + result = decoder.read_null() + elif writer_schema.type == 'boolean': + result = decoder.read_boolean() + elif writer_schema.type == 'string': + result = decoder.read_utf8() + elif writer_schema.type == 'int': + result = decoder.read_int() + elif writer_schema.type == 'long': + result = decoder.read_long() + elif writer_schema.type == 'float': + result = decoder.read_float() + elif writer_schema.type == 'double': + result = decoder.read_double() + elif writer_schema.type == 'bytes': + result = decoder.read_bytes() + elif writer_schema.type == 'fixed': + result = self.read_fixed(writer_schema, decoder) + elif writer_schema.type == 'enum': + result = self.read_enum(writer_schema, reader_schema, decoder) + elif writer_schema.type == 'array': + result = self.read_array(writer_schema, reader_schema, decoder) + elif writer_schema.type == 'map': + result = self.read_map(writer_schema, reader_schema, decoder) + elif writer_schema.type in ['union', 'error_union']: + result = self.read_union(writer_schema, reader_schema, decoder) + elif writer_schema.type in ['record', 'error', 'request']: + result = self.read_record(writer_schema, reader_schema, decoder) + else: + fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type + raise schema.AvroException(fail_msg) + return result + + def skip_data(self, writer_schema, decoder): + if writer_schema.type == 'null': + result = decoder.skip_null() + elif writer_schema.type == 'boolean': + result = decoder.skip_boolean() + elif writer_schema.type == 'string': + result = decoder.skip_utf8() + elif writer_schema.type == 'int': + result = decoder.skip_int() + elif writer_schema.type == 'long': + result = decoder.skip_long() + elif writer_schema.type == 'float': + result = decoder.skip_float() + elif writer_schema.type == 'double': + result = decoder.skip_double() + elif writer_schema.type == 'bytes': + result = decoder.skip_bytes() + elif writer_schema.type == 'fixed': + result = self.skip_fixed(writer_schema, decoder) + elif writer_schema.type == 'enum': + result = self.skip_enum(decoder) + elif writer_schema.type == 'array': + self.skip_array(writer_schema, decoder) + result = None + elif writer_schema.type == 'map': + self.skip_map(writer_schema, decoder) + result = None + elif writer_schema.type in ['union', 'error_union']: + result = self.skip_union(writer_schema, decoder) + elif writer_schema.type in ['record', 'error', 'request']: + self.skip_record(writer_schema, decoder) + result = None + else: + fail_msg = "Unknown schema type: %s" % writer_schema.type + raise schema.AvroException(fail_msg) + return result + + @staticmethod + def read_fixed(writer_schema, decoder): + """ + Fixed instances are encoded using the number of bytes declared + in the schema. + """ + return decoder.read(writer_schema.size) + + @staticmethod + def skip_fixed(writer_schema, decoder): + return decoder.skip(writer_schema.size) + + @staticmethod + def read_enum(writer_schema, reader_schema, decoder): + """ + An enum is encoded by a int, representing the zero-based position + of the symbol in the schema. + """ + # read data + index_of_symbol = decoder.read_int() + if index_of_symbol >= len(writer_schema.symbols): + fail_msg = "Can't access enum index %d for enum with %d symbols" \ + % (index_of_symbol, len(writer_schema.symbols)) + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + read_symbol = writer_schema.symbols[index_of_symbol] + + # schema resolution + if read_symbol not in reader_schema.symbols: + fail_msg = "Symbol %s not present in Reader's Schema" % read_symbol + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + return read_symbol + + @staticmethod + def skip_enum(decoder): + return decoder.skip_int() + + def read_array(self, writer_schema, reader_schema, decoder): + """ + Arrays are encoded as a series of blocks. + + Each block consists of a long count value, + followed by that many array items. + A block with count zero indicates the end of the array. + Each item is encoded per the array's item schema. + + If a block's count is negative, + then the count is followed immediately by a long block size, + indicating the number of bytes in the block. + The actual count in this case + is the absolute value of the count written. + """ + read_items = [] + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_count = -block_count + decoder.read_long() + for _ in range(block_count): + read_items.append(self.read_data(writer_schema.items, + reader_schema.items, decoder)) + block_count = decoder.read_long() + return read_items + + def skip_array(self, writer_schema, decoder): + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_size = decoder.read_long() + decoder.skip(block_size) + else: + for _ in range(block_count): + self.skip_data(writer_schema.items, decoder) + block_count = decoder.read_long() + + def read_map(self, writer_schema, reader_schema, decoder): + """ + Maps are encoded as a series of blocks. + + Each block consists of a long count value, + followed by that many key/value pairs. + A block with count zero indicates the end of the map. + Each item is encoded per the map's value schema. + + If a block's count is negative, + then the count is followed immediately by a long block size, + indicating the number of bytes in the block. + The actual count in this case + is the absolute value of the count written. + """ + read_items = {} + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_count = -block_count + decoder.read_long() + for _ in range(block_count): + key = decoder.read_utf8() + read_items[key] = self.read_data(writer_schema.values, + reader_schema.values, decoder) + block_count = decoder.read_long() + return read_items + + def skip_map(self, writer_schema, decoder): + block_count = decoder.read_long() + while block_count != 0: + if block_count < 0: + block_size = decoder.read_long() + decoder.skip(block_size) + else: + for _ in range(block_count): + decoder.skip_utf8() + self.skip_data(writer_schema.values, decoder) + block_count = decoder.read_long() + + def read_union(self, writer_schema, reader_schema, decoder): + """ + A union is encoded by first writing a long value indicating + the zero-based position within the union of the schema of its value. + The value is then encoded per the indicated schema within the union. + """ + # schema resolution + index_of_schema = int(decoder.read_long()) + if index_of_schema >= len(writer_schema.schemas): + fail_msg = "Can't access branch index %d for union with %d branches" \ + % (index_of_schema, len(writer_schema.schemas)) + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + selected_writer_schema = writer_schema.schemas[index_of_schema] + + # read data + return self.read_data(selected_writer_schema, reader_schema, decoder) + + def skip_union(self, writer_schema, decoder): + index_of_schema = int(decoder.read_long()) + if index_of_schema >= len(writer_schema.schemas): + fail_msg = "Can't access branch index %d for union with %d branches" \ + % (index_of_schema, len(writer_schema.schemas)) + raise SchemaResolutionException(fail_msg, writer_schema) + return self.skip_data(writer_schema.schemas[index_of_schema], decoder) + + def read_record(self, writer_schema, reader_schema, decoder): + """ + A record is encoded by encoding the values of its fields + in the order that they are declared. In other words, a record + is encoded as just the concatenation of the encodings of its fields. + Field values are encoded per their schema. + + Schema Resolution: + * the ordering of fields may be different: fields are matched by name. + * schemas for fields with the same name in both records are resolved + recursively. + * if the writer's record contains a field with a name not present in the + reader's record, the writer's value for that field is ignored. + * if the reader's record schema has a field that contains a default value, + and writer's schema does not have a field with the same name, then the + reader should use the default value from its field. + * if the reader's record schema has a field with no default value, and + writer's schema does not have a field with the same name, then the + field's value is unset. + """ + # schema resolution + readers_fields_dict = reader_schema.field_map + read_record = {} + for field in writer_schema.fields: + readers_field = readers_fields_dict.get(field.name) + if readers_field is not None: + field_val = self.read_data(field.type, readers_field.type, decoder) + read_record[field.name] = field_val + else: + self.skip_data(field.type, decoder) + + # fill in default values + if len(readers_fields_dict) > len(read_record): + writers_fields_dict = writer_schema.field_map + for field_name, field in readers_fields_dict.items(): + if field_name not in writers_fields_dict: + if field.has_default: + field_val = self._read_default_value(field.type, field.default) + read_record[field.name] = field_val + else: + fail_msg = 'No default value for field %s' % field_name + raise SchemaResolutionException(fail_msg, writer_schema, + reader_schema) + return read_record + + def skip_record(self, writer_schema, decoder): + for field in writer_schema.fields: + self.skip_data(field.type, decoder) + + def _read_default_value(self, field_schema, default_value): + """ + Basically a JSON Decoder? + """ + if field_schema.type == 'null': + result = None + elif field_schema.type == 'boolean': + result = bool(default_value) + elif field_schema.type == 'int': + result = int(default_value) + elif field_schema.type == 'long': + result = int(default_value) + elif field_schema.type in ['float', 'double']: + result = float(default_value) + elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']: + result = default_value + elif field_schema.type == 'array': + read_array = [] + for json_val in default_value: + item_val = self._read_default_value(field_schema.items, json_val) + read_array.append(item_val) + result = read_array + elif field_schema.type == 'map': + read_map = {} + for key, json_val in default_value.items(): + map_val = self._read_default_value(field_schema.values, json_val) + read_map[key] = map_val + result = read_map + elif field_schema.type in ['union', 'error_union']: + result = self._read_default_value(field_schema.schemas[0], default_value) + elif field_schema.type == 'record': + read_record = {} + for field in field_schema.fields: + json_val = default_value.get(field.name) + if json_val is None: + json_val = field.default + field_val = self._read_default_value(field.type, json_val) + read_record[field.name] = field_val + result = read_record + else: + fail_msg = 'Unknown type: %s' % field_schema.type + raise schema.AvroException(fail_msg) + return result # ------------------------------------------------------------------------------ if __name__ == '__main__': - raise Exception('Not a standalone module') + raise Exception('Not a standalone module') diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py index 159607e06120..6e0581df262b 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py @@ -1,22 +1,8 @@ -#!/usr/bin/env python3 -# -*- mode: python -*- -# -*- coding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- """Read/Write Avro File Object Containers.""" @@ -39,11 +25,11 @@ VERSION = 1 if PY3: - MAGIC = b'Obj' + bytes([VERSION]) - MAGIC_SIZE = len(MAGIC) + MAGIC = b'Obj' + bytes([VERSION]) + MAGIC_SIZE = len(MAGIC) else: - MAGIC = 'Obj' + chr(VERSION) - MAGIC_SIZE = len(MAGIC) + MAGIC = 'Obj' + chr(VERSION) + MAGIC_SIZE = len(MAGIC) # Size of the synchronization marker, in number of bytes: SYNC_SIZE = 16 @@ -90,212 +76,203 @@ class DataFileException(schema.AvroException): - """Problem reading or writing file object containers.""" - - def __init__(self, msg): - super(DataFileException, self).__init__(msg) + """Problem reading or writing file object containers.""" # ------------------------------------------------------------------------------ class DataFileReader(object): - """Read files written by DataFileWriter.""" - - # TODO: allow user to specify expected schema? - # TODO: allow user to specify the encoder - def __init__(self, reader, datum_reader): - """Initializes a new data file reader. - - Args: - reader: Open file to read from. - datum_reader: Avro datum reader. - """ - self._reader = reader - self._raw_decoder = avro_io.BinaryDecoder(reader) - self._datum_decoder = None # Maybe reset at every block. - self._datum_reader = datum_reader - - # read the header: magic, meta, sync - self._read_header() - - # ensure codec is valid - avro_codec_raw = self.GetMeta('avro.codec') - if avro_codec_raw is None: - self.codec = "null" - else: - self.codec = avro_codec_raw.decode('utf-8') - if self.codec not in VALID_CODECS: - raise DataFileException('Unknown codec: %s.' % self.codec) - - self._file_length = self._GetInputFileLength() - - # get ready to read - self._block_count = 0 - self.datum_reader.writer_schema = ( - schema.parse(self.GetMeta(SCHEMA_KEY).decode('utf-8'))) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - # Perform a close if there's no exception - if type is None: - self.close() - - def __iter__(self): - return self - - def __next__(self): - """Implements the iterator interface.""" - return next(self) - - # read-only properties - @property - def reader(self): - return self._reader - - @property - def raw_decoder(self): - return self._raw_decoder - - @property - def datum_decoder(self): - return self._datum_decoder - - @property - def datum_reader(self): - return self._datum_reader - - @property - def sync_marker(self): - return self._sync_marker - - @property - def meta(self): - return self._meta - - @property - def file_length(self): - """Length of the input file, in bytes.""" - return self._file_length - - # read/write properties - @property - def block_count(self): - return self._block_count - - def GetMeta(self, key): - """Reports the value of a given metadata key. - - Args: - key: Metadata key (string) to report the value of. - Returns: - Value associated to the metadata key, as bytes. - """ - return self._meta.get(key) - - def SetMeta(self, key, value): - """Sets a metadata. - - Args: - key: Metadata key (string) to set. - value: Metadata value to set, as bytes. - """ - if isinstance(value, str): - value = value.encode('utf-8') - self._meta[key] = value - - def _GetInputFileLength(self): - """Reports the length of the input file, in bytes. - - Leaves the current position unmodified. - - Returns: - The length of the input file, in bytes. - """ - current_pos = self.reader.tell() - self.reader.seek(0, 2) - file_length = self.reader.tell() - self.reader.seek(current_pos) - return file_length - - def is_EOF(self): - return self.reader.tell() == self.file_length - - def _read_header(self): - # seek to the beginning of the file to get magic block - self.reader.seek(0, 0) - - # read header into a dict - header = self.datum_reader.read_data( - META_SCHEMA, META_SCHEMA, self.raw_decoder) - - # check magic number - if header.get('magic') != MAGIC: - fail_msg = "Not an Avro data file: %s doesn't match %s."\ - % (header.get('magic'), MAGIC) - raise schema.AvroException(fail_msg) - - # set metadata - self._meta = header['meta'] - - # set sync marker - self._sync_marker = header['sync'] - - def _read_block_header(self): - self._block_count = self.raw_decoder.read_long() - if self.codec == "null": - # Skip a long; we don't need to use the length. - self.raw_decoder.skip_long() - self._datum_decoder = self._raw_decoder - elif self.codec == 'deflate': - # Compressed data is stored as (length, data), which - # corresponds to how the "bytes" type is encoded. - data = self.raw_decoder.read_bytes() - # -15 is the log of the window size; negative indicates - # "raw" (no zlib headers) decompression. See zlib.h. - uncompressed = zlib.decompress(data, -15) - self._datum_decoder = avro_io.BinaryDecoder(io.BytesIO(uncompressed)) - else: - raise DataFileException("Unknown codec: %r" % self.codec) - - def _skip_sync(self): - """ - Read the length of the sync marker; if it matches the sync marker, - return True. Otherwise, seek back to where we started and return False. - """ - proposed_sync_marker = self.reader.read(SYNC_SIZE) - if proposed_sync_marker != self.sync_marker: - self.reader.seek(-SYNC_SIZE, 1) - return False - else: - return True - - # TODO: handle block of length zero - # TODO: clean this up with recursion - def __next__(self): - """Return the next datum in the file.""" - if self.block_count == 0: - if self.is_EOF(): - raise StopIteration - elif self._skip_sync(): - if self.is_EOF(): raise StopIteration - self._read_block_header() - else: - self._read_block_header() - - datum = self.datum_reader.read(self.datum_decoder) - self._block_count -= 1 - return datum - - # PY2 - def next(self): - return self.__next__() - - def close(self): - """Close this reader.""" - self.reader.close() + """Read files written by DataFileWriter.""" + + # TODO: allow user to specify expected schema? + # TODO: allow user to specify the encoder + def __init__(self, reader, datum_reader): + """Initializes a new data file reader. + + Args: + reader: Open file to read from. + datum_reader: Avro datum reader. + """ + self._reader = reader + self._raw_decoder = avro_io.BinaryDecoder(reader) + self._datum_decoder = None # Maybe reset at every block. + self._datum_reader = datum_reader + + # read the header: magic, meta, sync + self._read_header() + + # ensure codec is valid + avro_codec_raw = self.GetMeta('avro.codec') + if avro_codec_raw is None: + self.codec = "null" + else: + self.codec = avro_codec_raw.decode('utf-8') + if self.codec not in VALID_CODECS: + raise DataFileException('Unknown codec: %s.' % self.codec) + + self._file_length = self._GetInputFileLength() + + # get ready to read + self._block_count = 0 + self.datum_reader.writer_schema = ( + schema.parse(self.GetMeta(SCHEMA_KEY).decode('utf-8'))) + + def __enter__(self): + return self + + def __exit__(self, data_type, value, traceback): + # Perform a close if there's no exception + if data_type is None: + self.close() + + def __iter__(self): + return self + + # read-only properties + @property + def reader(self): + return self._reader + + @property + def raw_decoder(self): + return self._raw_decoder + + @property + def datum_decoder(self): + return self._datum_decoder + + @property + def datum_reader(self): + return self._datum_reader + + @property + def sync_marker(self): + return self._sync_marker + + @property + def meta(self): + return self._meta + + @property + def file_length(self): + """Length of the input file, in bytes.""" + return self._file_length + + # read/write properties + @property + def block_count(self): + return self._block_count + + def GetMeta(self, key): + """Reports the value of a given metadata key. + + Args: + key: Metadata key (string) to report the value of. + Returns: + Value associated to the metadata key, as bytes. + """ + return self._meta.get(key) + + def SetMeta(self, key, value): + """Sets a metadata. + + Args: + key: Metadata key (string) to set. + value: Metadata value to set, as bytes. + """ + if isinstance(value, str): + value = value.encode('utf-8') + self._meta[key] = value + + def _GetInputFileLength(self): + """Reports the length of the input file, in bytes. + + Leaves the current position unmodified. + + Returns: + The length of the input file, in bytes. + """ + current_pos = self.reader.tell() + self.reader.seek(0, 2) + file_length = self.reader.tell() + self.reader.seek(current_pos) + return file_length + + def is_EOF(self): + return self.reader.tell() == self.file_length + + def _read_header(self): + # seek to the beginning of the file to get magic block + self.reader.seek(0, 0) + + # read header into a dict + header = self.datum_reader.read_data( + META_SCHEMA, META_SCHEMA, self.raw_decoder) + + # check magic number + if header.get('magic') != MAGIC: + fail_msg = "Not an Avro data file: %s doesn't match %s." \ + % (header.get('magic'), MAGIC) + raise schema.AvroException(fail_msg) + + # set metadata + self._meta = header['meta'] + + # set sync marker + self._sync_marker = header['sync'] + + def _read_block_header(self): + self._block_count = self.raw_decoder.read_long() + if self.codec == "null": + # Skip a long; we don't need to use the length. + self.raw_decoder.skip_long() + self._datum_decoder = self._raw_decoder + elif self.codec == 'deflate': + # Compressed data is stored as (length, data), which + # corresponds to how the "bytes" type is encoded. + data = self.raw_decoder.read_bytes() + # -15 is the log of the window size; negative indicates + # "raw" (no zlib headers) decompression. See zlib.h. + uncompressed = zlib.decompress(data, -15) + self._datum_decoder = avro_io.BinaryDecoder(io.BytesIO(uncompressed)) + else: + raise DataFileException("Unknown codec: %r" % self.codec) + + def _skip_sync(self): + """ + Read the length of the sync marker; if it matches the sync marker, + return True. Otherwise, seek back to where we started and return False. + """ + proposed_sync_marker = self.reader.read(SYNC_SIZE) + if proposed_sync_marker != self.sync_marker: + self.reader.seek(-SYNC_SIZE, 1) + return False + return True + + def __next__(self): + """Return the next datum in the file.""" + if self.block_count == 0: + if self.is_EOF(): + raise StopIteration + if self._skip_sync(): + if self.is_EOF(): + raise StopIteration + self._read_block_header() + else: + self._read_block_header() + + datum = self.datum_reader.read(self.datum_decoder) + self._block_count -= 1 + return datum + + # PY2 + def next(self): + return self.__next__() + + def close(self): + """Close this reader.""" + self.reader.close() if __name__ == '__main__': - raise Exception('Not a standalone module') + raise Exception('Not a standalone module') diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py index 65dfdc7a78a2..a18f79bce677 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py @@ -1,22 +1,9 @@ -#!/usr/bin/env python3 -# -*- mode: python -*- -# -*- coding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +# pylint: disable=too-many-lines """Representation of Avro schemas. @@ -43,13 +30,14 @@ import logging import re import sys -import warnings from six import with_metaclass PY2 = sys.version_info[0] == 2 if PY2: - str = unicode + _str = unicode # pylint: disable=undefined-variable +else: + _str = str logger = logging.getLogger(__name__) @@ -57,83 +45,82 @@ # Constants # Log level more verbose than DEBUG=10, INFO=20, etc. -DEBUG_VERBOSE=5 +DEBUG_VERBOSE = 5 - -NULL = 'null' +NULL = 'null' BOOLEAN = 'boolean' -STRING = 'string' -BYTES = 'bytes' -INT = 'int' -LONG = 'long' -FLOAT = 'float' -DOUBLE = 'double' -FIXED = 'fixed' -ENUM = 'enum' -RECORD = 'record' -ERROR = 'error' -ARRAY = 'array' -MAP = 'map' -UNION = 'union' +STRING = 'string' +BYTES = 'bytes' +INT = 'int' +LONG = 'long' +FLOAT = 'float' +DOUBLE = 'double' +FIXED = 'fixed' +ENUM = 'enum' +RECORD = 'record' +ERROR = 'error' +ARRAY = 'array' +MAP = 'map' +UNION = 'union' # Request and error unions are part of Avro protocols: REQUEST = 'request' ERROR_UNION = 'error_union' PRIMITIVE_TYPES = frozenset([ - NULL, - BOOLEAN, - STRING, - BYTES, - INT, - LONG, - FLOAT, - DOUBLE, + NULL, + BOOLEAN, + STRING, + BYTES, + INT, + LONG, + FLOAT, + DOUBLE, ]) NAMED_TYPES = frozenset([ - FIXED, - ENUM, - RECORD, - ERROR, + FIXED, + ENUM, + RECORD, + ERROR, ]) VALID_TYPES = frozenset.union( - PRIMITIVE_TYPES, - NAMED_TYPES, - [ - ARRAY, - MAP, - UNION, - REQUEST, - ERROR_UNION, - ], + PRIMITIVE_TYPES, + NAMED_TYPES, + [ + ARRAY, + MAP, + UNION, + REQUEST, + ERROR_UNION, + ], ) SCHEMA_RESERVED_PROPS = frozenset([ - 'type', - 'name', - 'namespace', - 'fields', # Record - 'items', # Array - 'size', # Fixed - 'symbols', # Enum - 'values', # Map - 'doc', + 'type', + 'name', + 'namespace', + 'fields', # Record + 'items', # Array + 'size', # Fixed + 'symbols', # Enum + 'values', # Map + 'doc', ]) FIELD_RESERVED_PROPS = frozenset([ - 'default', - 'name', - 'doc', - 'order', - 'type', + 'default', + 'name', + 'doc', + 'order', + 'type', ]) VALID_FIELD_SORT_ORDERS = frozenset([ - 'ascending', - 'descending', - 'ignore', + 'ascending', + 'descending', + 'ignore', ]) @@ -142,87 +129,84 @@ class Error(Exception): - """Base class for errors in this module.""" - pass + """Base class for errors in this module.""" class AvroException(Error): - """Generic Avro schema error.""" - pass + """Generic Avro schema error.""" class SchemaParseException(AvroException): - """Error while parsing a JSON schema descriptor.""" - pass + """Error while parsing a JSON schema descriptor.""" class Schema(with_metaclass(abc.ABCMeta, object)): - """Abstract base class for all Schema classes.""" + """Abstract base class for all Schema classes.""" - def __init__(self, type, other_props=None): - """Initializes a new schema object. + def __init__(self, data_type, other_props=None): + """Initializes a new schema object. - Args: - type: Type of the schema to initialize. - other_props: Optional dictionary of additional properties. - """ - if type not in VALID_TYPES: - raise SchemaParseException('%r is not a valid Avro type.' % type) + Args: + data_type: Type of the schema to initialize. + other_props: Optional dictionary of additional properties. + """ + if data_type not in VALID_TYPES: + raise SchemaParseException('%r is not a valid Avro type.' % data_type) - # All properties of this schema, as a map: property name -> property value - self._props = {} + # All properties of this schema, as a map: property name -> property value + self._props = {} - self._props['type'] = type - self._type = type + self._props['type'] = data_type + self._type = data_type - if other_props: - self._props.update(other_props) + if other_props: + self._props.update(other_props) - @property - def namespace(self): - """Returns: the namespace this schema belongs to, if any, or None.""" - return self._props.get('namespace', None) + @property + def namespace(self): + """Returns: the namespace this schema belongs to, if any, or None.""" + return self._props.get('namespace', None) - @property - def type(self): - """Returns: the type of this schema.""" - return self._type + @property + def type(self): + """Returns: the type of this schema.""" + return self._type - @property - def doc(self): - """Returns: the documentation associated to this schema, if any, or None.""" - return self._props.get('doc', None) + @property + def doc(self): + """Returns: the documentation associated to this schema, if any, or None.""" + return self._props.get('doc', None) - @property - def props(self): - """Reports all the properties of this schema. + @property + def props(self): + """Reports all the properties of this schema. - Includes all properties, reserved and non reserved. - JSON properties of this schema are directly generated from this dict. + Includes all properties, reserved and non reserved. + JSON properties of this schema are directly generated from this dict. - Returns: - A dictionary of properties associated to this schema. - """ - return self._props + Returns: + A dictionary of properties associated to this schema. + """ + return self._props - @property - def other_props(self): - """Returns: the dictionary of non-reserved properties.""" - return dict(FilterKeysOut(items=self._props, keys=SCHEMA_RESERVED_PROPS)) + @property + def other_props(self): + """Returns: the dictionary of non-reserved properties.""" + return dict(FilterKeysOut(items=self._props, keys=SCHEMA_RESERVED_PROPS)) - def __str__(self): - """Returns: the JSON representation of this schema.""" - return json.dumps(self.to_json()) + def __str__(self): + """Returns: the JSON representation of this schema.""" + return json.dumps(self.to_json(names=None)) - @abc.abstractmethod - def to_json(self, names): - """Converts the schema object into its AVRO specification representation. + @abc.abstractmethod + def to_json(self, names): + """Converts the schema object into its AVRO specification representation. - Schema types that have names (records, enums, and fixed) must - be aware of not re-defining schemas that are already listed - in the parameter names. - """ - raise Exception('Cannot run abstract method.') + Schema types that have names (records, enums, and fixed) must + be aware of not re-defining schemas that are already listed + in the parameter names. + """ + raise Exception('Cannot run abstract method.') # ------------------------------------------------------------------------------ @@ -231,244 +215,255 @@ def to_json(self, names): _RE_NAME = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') _RE_FULL_NAME = re.compile( - r'^' - r'[.]?(?:[A-Za-z_][A-Za-z0-9_]*[.])*' # optional namespace - r'([A-Za-z_][A-Za-z0-9_]*)' # name - r'$' + r'^' + r'[.]?(?:[A-Za-z_][A-Za-z0-9_]*[.])*' # optional namespace + r'([A-Za-z_][A-Za-z0-9_]*)' # name + r'$' ) -class Name(object): - """Representation of an Avro name.""" - - def __init__(self, name, namespace=None): - """Parses an Avro name. - - Args: - name: Avro name to parse (relative or absolute). - namespace: Optional explicit namespace if the name is relative. - """ - # Normalize: namespace is always defined as a string, possibly empty. - if namespace is None: namespace = '' - - if '.' in name: - # name is absolute, namespace is ignored: - self._fullname = name - match = _RE_FULL_NAME.match(self._fullname) - if match is None: - raise SchemaParseException( - 'Invalid absolute schema name: %r.' % self._fullname) - - self._name = match.group(1) - self._namespace = self._fullname[:-(len(self._name) + 1)] - - else: - # name is relative, combine with explicit namespace: - self._name = name - self._namespace = namespace - self._fullname = (self._name - if (not self._namespace) else - '%s.%s' % (self._namespace, self._name)) - - # Validate the fullname: - if _RE_FULL_NAME.match(self._fullname) is None: - raise SchemaParseException( - 'Invalid schema name %r infered from name %r and namespace %r.' - % (self._fullname, self._name, self._namespace)) - - def __eq__(self, other): - if not isinstance(other, Name): - return NotImplemented - return self.fullname == other.fullname - - @property - def simple_name(self): - """Returns: the simple name part of this name.""" - return self._name - - @property - def namespace(self): - """Returns: this name's namespace, possible the empty string.""" - return self._namespace - - @property - def fullname(self): - """Returns: the full name.""" - return self._fullname +class Name(object): + """Representation of an Avro name.""" + + def __init__(self, name, namespace=None): + """Parses an Avro name. + + Args: + name: Avro name to parse (relative or absolute). + namespace: Optional explicit namespace if the name is relative. + """ + # Normalize: namespace is always defined as a string, possibly empty. + if namespace is None: + namespace = '' + + if '.' in name: + # name is absolute, namespace is ignored: + self._fullname = name + + match = _RE_FULL_NAME.match(self._fullname) + if match is None: + raise SchemaParseException( + 'Invalid absolute schema name: %r.' % self._fullname) + + self._name = match.group(1) + self._namespace = self._fullname[:-(len(self._name) + 1)] + + else: + # name is relative, combine with explicit namespace: + self._name = name + self._namespace = namespace + self._fullname = (self._name + if (not self._namespace) else + '%s.%s' % (self._namespace, self._name)) + + # Validate the fullname: + if _RE_FULL_NAME.match(self._fullname) is None: + raise SchemaParseException( + 'Invalid schema name %r infered from name %r and namespace %r.' + % (self._fullname, self._name, self._namespace)) + + def __eq__(self, other): + if not isinstance(other, Name): + return NotImplemented + return self.fullname == other.fullname + + @property + def simple_name(self): + """Returns: the simple name part of this name.""" + return self._name + + @property + def namespace(self): + """Returns: this name's namespace, possible the empty string.""" + return self._namespace + + @property + def fullname(self): + """Returns: the full name.""" + return self._fullname # ------------------------------------------------------------------------------ class Names(object): - """Tracks Avro named schemas and default namespace during parsing.""" - - def __init__(self, default_namespace=None, names=None): - """Initializes a new name tracker. - - Args: - default_namespace: Optional default namespace. - names: Optional initial mapping of known named schemas. - """ - if names is None: - names = {} - self._names = names - self._default_namespace = default_namespace - - @property - def names(self): - """Returns: the mapping of known named schemas.""" - return self._names - - @property - def default_namespace(self): - """Returns: the default namespace, if any, or None.""" - return self._default_namespace - - def NewWithDefaultNamespace(self, namespace): - """Creates a new name tracker from this tracker, but with a new default ns. - - Args: - namespace: New default namespace to use. - Returns: - New name tracker with the specified default namespace. - """ - return Names(names=self._names, default_namespace=namespace) - - def GetName(self, name, namespace=None): - """Resolves the Avro name according to this name tracker's state. - - Args: - name: Name to resolve (absolute or relative). - namespace: Optional explicit namespace. - Returns: - The specified name, resolved according to this tracker. - """ - if namespace is None: namespace = self._default_namespace - return Name(name=name, namespace=namespace) - - def has_name(self, name, namespace=None): - avro_name = self.GetName(name=name, namespace=namespace) - return avro_name.fullname in self._names - - def get_name(self, name, namespace=None): - avro_name = self.GetName(name=name, namespace=namespace) - return self._names.get(avro_name.fullname, None) - - def GetSchema(self, name, namespace=None): - """Resolves an Avro schema by name. - - Args: - name: Name (relative or absolute) of the Avro schema to look up. - namespace: Optional explicit namespace. - Returns: - The schema with the specified name, if any, or None. - """ - avro_name = self.GetName(name=name, namespace=namespace) - return self._names.get(avro_name.fullname, None) - - def prune_namespace(self, properties): - """given a properties, return properties with namespace removed if - it matches the own default namespace - """ - if self.default_namespace is None: - # I have no default -- no change - return properties - if 'namespace' not in properties: - # he has no namespace - no change - return properties - if properties['namespace'] != self.default_namespace: - # we're different - leave his stuff alone - return properties - # we each have a namespace and it's redundant. delete his. - prunable = properties.copy() - del(prunable['namespace']) - return prunable - - def Register(self, schema): - """Registers a new named schema in this tracker. - - Args: - schema: Named Avro schema to register in this tracker. - """ - if schema.fullname in VALID_TYPES: - raise SchemaParseException( - '%s is a reserved type name.' % schema.fullname) - if schema.fullname in self.names: - raise SchemaParseException( - 'Avro name %r already exists.' % schema.fullname) - - logger.log(DEBUG_VERBOSE, 'Register new name for %r', schema.fullname) - self._names[schema.fullname] = schema + """Tracks Avro named schemas and default namespace during parsing.""" + + def __init__(self, default_namespace=None, names=None): + """Initializes a new name tracker. + + Args: + default_namespace: Optional default namespace. + names: Optional initial mapping of known named schemas. + """ + if names is None: + names = {} + self._names = names + self._default_namespace = default_namespace + + @property + def names(self): + """Returns: the mapping of known named schemas.""" + return self._names + + @property + def default_namespace(self): + """Returns: the default namespace, if any, or None.""" + return self._default_namespace + + def NewWithDefaultNamespace(self, namespace): + """Creates a new name tracker from this tracker, but with a new default ns. + + Args: + namespace: New default namespace to use. + Returns: + New name tracker with the specified default namespace. + """ + return Names(names=self._names, default_namespace=namespace) + + def GetName(self, name, namespace=None): + """Resolves the Avro name according to this name tracker's state. + + Args: + name: Name to resolve (absolute or relative). + namespace: Optional explicit namespace. + Returns: + The specified name, resolved according to this tracker. + """ + if namespace is None: + namespace = self._default_namespace + return Name(name=name, namespace=namespace) + + def has_name(self, name, namespace=None): + avro_name = self.GetName(name=name, namespace=namespace) + return avro_name.fullname in self._names + + def get_name(self, name, namespace=None): + avro_name = self.GetName(name=name, namespace=namespace) + return self._names.get(avro_name.fullname, None) + + def GetSchema(self, name, namespace=None): + """Resolves an Avro schema by name. + + Args: + name: Name (relative or absolute) of the Avro schema to look up. + namespace: Optional explicit namespace. + Returns: + The schema with the specified name, if any, or None. + """ + avro_name = self.GetName(name=name, namespace=namespace) + return self._names.get(avro_name.fullname, None) + + def prune_namespace(self, properties): + """given a properties, return properties with namespace removed if + it matches the own default namespace + """ + if self.default_namespace is None: + # I have no default -- no change + return properties + if 'namespace' not in properties: + # he has no namespace - no change + return properties + if properties['namespace'] != self.default_namespace: + # we're different - leave his stuff alone + return properties + # we each have a namespace and it's redundant. delete his. + prunable = properties.copy() + del prunable['namespace'] + return prunable + + def Register(self, schema): + """Registers a new named schema in this tracker. + + Args: + schema: Named Avro schema to register in this tracker. + """ + if schema.fullname in VALID_TYPES: + raise SchemaParseException( + '%s is a reserved type name.' % schema.fullname) + if schema.fullname in self.names: + raise SchemaParseException( + 'Avro name %r already exists.' % schema.fullname) + + logger.log(DEBUG_VERBOSE, 'Register new name for %r', schema.fullname) + self._names[schema.fullname] = schema # ------------------------------------------------------------------------------ class NamedSchema(Schema): - """Abstract base class for named schemas. - - Named schemas are enumerated in NAMED_TYPES. - """ + """Abstract base class for named schemas. - def __init__( - self, - type, - name, - namespace=None, - names=None, - other_props=None, - ): - """Initializes a new named schema object. - - Args: - type: Type of the named schema. - name: Name (absolute or relative) of the schema. - namespace: Optional explicit namespace if name is relative. - names: Tracker to resolve and register Avro names. - other_props: Optional map of additional properties of the schema. + Named schemas are enumerated in NAMED_TYPES. """ - assert (type in NAMED_TYPES), ('Invalid named type: %r' % type) - self._avro_name = names.GetName(name=name, namespace=namespace) - - super(NamedSchema, self).__init__(type, other_props) - - names.Register(self) - - self._props['name'] = self.name - if self.namespace: - self._props['namespace'] = self.namespace - - @property - def avro_name(self): - """Returns: the Name object describing this schema's name.""" - return self._avro_name - - @property - def name(self): - return self._avro_name.simple_name - - @property - def namespace(self): - return self._avro_name.namespace - - @property - def fullname(self): - return self._avro_name.fullname - - def name_ref(self, names): - """Reports this schema name relative to the specified name tracker. - - Args: - names: Avro name tracker to relativise this schema name against. - Returns: - This schema name, relativised against the specified name tracker. - """ - if self.namespace == names.default_namespace: - return self.name - else: - return self.fullname + def __init__( + self, + data_type, + name, + namespace=None, + names=None, + other_props=None, + ): + """Initializes a new named schema object. + + Args: + data_type: Type of the named schema. + name: Name (absolute or relative) of the schema. + namespace: Optional explicit namespace if name is relative. + names: Tracker to resolve and register Avro names. + other_props: Optional map of additional properties of the schema. + """ + assert (data_type in NAMED_TYPES), ('Invalid named type: %r' % data_type) + self._avro_name = names.GetName(name=name, namespace=namespace) + + super(NamedSchema, self).__init__(data_type, other_props) + + names.Register(self) + + self._props['name'] = self.name + if self.namespace: + self._props['namespace'] = self.namespace + + @property + def avro_name(self): + """Returns: the Name object describing this schema's name.""" + return self._avro_name + + @property + def name(self): + return self._avro_name.simple_name + + @property + def namespace(self): + return self._avro_name.namespace + + @property + def fullname(self): + return self._avro_name.fullname + + def name_ref(self, names): + """Reports this schema name relative to the specified name tracker. + + Args: + names: Avro name tracker to relativise this schema name against. + Returns: + This schema name, relativised against the specified name tracker. + """ + if self.namespace == names.default_namespace: + return self.name + return self.fullname + + @abc.abstractmethod + def to_json(self, names): + """Converts the schema object into its AVRO specification representation. + + Schema types that have names (records, enums, and fixed) must + be aware of not re-defining schemas that are already listed + in the parameter names. + """ + raise Exception('Cannot run abstract method.') # ------------------------------------------------------------------------------ @@ -477,111 +472,109 @@ def name_ref(self, names): class Field(object): - """Representation of the schema of a field in a record.""" - - def __init__( - self, - type, - name, - index, - has_default, - default=_NO_DEFAULT, - order=None, - names=None, - doc=None, - other_props=None - ): - """Initializes a new Field object. - - Args: - type: Avro schema of the field. - name: Name of the field. - index: 0-based position of the field. - has_default: - default: - order: - names: - doc: - other_props: - """ - if (not isinstance(name, str)) or (len(name) == 0): - raise SchemaParseException('Invalid record field name: %r.' % name) - if (order is not None) and (order not in VALID_FIELD_SORT_ORDERS): - raise SchemaParseException('Invalid record field order: %r.' % order) - - # All properties of this record field: - self._props = {} - - self._has_default = has_default - if other_props: - self._props.update(other_props) - - self._index = index - self._type = self._props['type'] = type - self._name = self._props['name'] = name - - # TODO: check to ensure default is valid - if has_default: - self._props['default'] = default - - if order is not None: - self._props['order'] = order - - if doc is not None: - self._props['doc'] = doc - - @property - def type(self): - """Returns: the schema of this field.""" - return self._type - - @property - def name(self): - """Returns: this field name.""" - return self._name - - @property - def index(self): - """Returns: the 0-based index of this field in the record.""" - return self._index - - @property - def default(self): - return self._props['default'] - - @property - def has_default(self): - return self._has_default - - @property - def order(self): - return self._props.get('order', None) - - @property - def doc(self): - return self._props.get('doc', None) - - @property - def props(self): - return self._props - - @property - def other_props(self): - return FilterKeysOut(items=self._props, keys=FIELD_RESERVED_PROPS) - - def __str__(self): - return json.dumps(self.to_json(), cls=MappingProxyEncoder) - - def to_json(self, names=None): - if names is None: - names = Names() - to_dump = self.props.copy() - to_dump['type'] = self.type.to_json(names) - return to_dump - - def __eq__(self, that): - to_cmp = json.loads(str(self)) - return to_cmp == json.loads(str(that)) + """Representation of the schema of a field in a record.""" + + def __init__( + self, + data_type, + name, + index, + has_default, + default=_NO_DEFAULT, + order=None, + doc=None, + other_props=None + ): + """Initializes a new Field object. + + Args: + data_type: Avro schema of the field. + name: Name of the field. + index: 0-based position of the field. + has_default: + default: + order: + doc: + other_props: + """ + if (not isinstance(name, _str)) or (not name): + raise SchemaParseException('Invalid record field name: %r.' % name) + if (order is not None) and (order not in VALID_FIELD_SORT_ORDERS): + raise SchemaParseException('Invalid record field order: %r.' % order) + + # All properties of this record field: + self._props = {} + + self._has_default = has_default + if other_props: + self._props.update(other_props) + + self._index = index + self._type = self._props['type'] = data_type + self._name = self._props['name'] = name + + # TODO: check to ensure default is valid + if has_default: + self._props['default'] = default + + if order is not None: + self._props['order'] = order + + if doc is not None: + self._props['doc'] = doc + + @property + def type(self): + """Returns: the schema of this field.""" + return self._type + + @property + def name(self): + """Returns: this field name.""" + return self._name + + @property + def index(self): + """Returns: the 0-based index of this field in the record.""" + return self._index + + @property + def default(self): + return self._props['default'] + + @property + def has_default(self): + return self._has_default + + @property + def order(self): + return self._props.get('order', None) + + @property + def doc(self): + return self._props.get('doc', None) + + @property + def props(self): + return self._props + + @property + def other_props(self): + return FilterKeysOut(items=self._props, keys=FIELD_RESERVED_PROPS) + + def __str__(self): + return json.dumps(self.to_json()) + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = self.props.copy() + to_dump['type'] = self.type.to_json(names) + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(_str(self)) + return to_cmp == json.loads(_str(that)) # ------------------------------------------------------------------------------ @@ -589,41 +582,40 @@ def __eq__(self, that): class PrimitiveSchema(Schema): - """Schema of a primitive Avro type. + """Schema of a primitive Avro type. - Valid primitive types are defined in PRIMITIVE_TYPES. - """ + Valid primitive types are defined in PRIMITIVE_TYPES. + """ - def __init__(self, type, other_props=None): - """Initializes a new schema object for the specified primitive type. + def __init__(self, data_type, other_props=None): + """Initializes a new schema object for the specified primitive type. - Args: - type: Type of the schema to construct. Must be primitive. - """ - if type not in PRIMITIVE_TYPES: - raise AvroException('%r is not a valid primitive type.' % type) - super(PrimitiveSchema, self).__init__(type, other_props=other_props) - - @property - def name(self): - """Returns: the simple name of this schema.""" - # The name of a primitive type is the type itself. - return self.type - - @property - def fullname(self): - """Returns: the fully qualified name of this schema.""" - # The full name is the simple name for primitive schema. - return self.name - - def to_json(self, names=None): - if len(self.props) == 1: - return self.fullname - else: - return self.props + Args: + data_type: Type of the schema to construct. Must be primitive. + """ + if data_type not in PRIMITIVE_TYPES: + raise AvroException('%r is not a valid primitive type.' % data_type) + super(PrimitiveSchema, self).__init__(data_type, other_props=other_props) + + @property + def name(self): + """Returns: the simple name of this schema.""" + # The name of a primitive type is the type itself. + return self.type + + @property + def fullname(self): + """Returns: the fully qualified name of this schema.""" + # The full name is the simple name for primitive schema. + return self.name - def __eq__(self, that): - return self.props == that.props + def to_json(self, names=None): + if len(self.props) == 1: + return self.fullname + return self.props + + def __eq__(self, that): + return self.props == that.props # ------------------------------------------------------------------------------ @@ -631,104 +623,102 @@ def __eq__(self, that): class FixedSchema(NamedSchema): - def __init__( - self, - name, - namespace, - size, - names=None, - other_props=None, - ): - # Ensure valid ctor args - if not isinstance(size, int): - fail_msg = 'Fixed Schema requires a valid integer for size property.' - raise AvroException(fail_msg) - - super(FixedSchema, self).__init__( - type=FIXED, - name=name, - namespace=namespace, - names=names, - other_props=other_props, - ) - self._props['size'] = size - - @property - def size(self): - """Returns: the size of this fixed schema, in bytes.""" - return self._props['size'] - - def to_json(self, names=None): - if names is None: - names = Names() - if self.fullname in names.names: - return self.name_ref(names) - else: - names.names[self.fullname] = self - return names.prune_namespace(self.props) - - def __eq__(self, that): - return self.props == that.props + def __init__( + self, + name, + namespace, + size, + names=None, + other_props=None, + ): + # Ensure valid ctor args + if not isinstance(size, int): + fail_msg = 'Fixed Schema requires a valid integer for size property.' + raise AvroException(fail_msg) + + super(FixedSchema, self).__init__( + data_type=FIXED, + name=name, + namespace=namespace, + names=names, + other_props=other_props, + ) + self._props['size'] = size + + @property + def size(self): + """Returns: the size of this fixed schema, in bytes.""" + return self._props['size'] + + def to_json(self, names=None): + if names is None: + names = Names() + if self.fullname in names.names: + return self.name_ref(names) + names.names[self.fullname] = self + return names.prune_namespace(self.props) + + def __eq__(self, that): + return self.props == that.props # ------------------------------------------------------------------------------ class EnumSchema(NamedSchema): - def __init__( - self, - name, - namespace, - symbols, - names=None, - doc=None, - other_props=None, - ): - """Initializes a new enumeration schema object. - - Args: - name: Simple name of this enumeration. - namespace: Optional namespace. - symbols: Ordered list of symbols defined in this enumeration. - names: - doc: - other_props: - """ - symbols = tuple(symbols) - symbol_set = frozenset(symbols) - if (len(symbol_set) != len(symbols) - or not all(map(lambda symbol: isinstance(symbol, str), symbols))): - raise AvroException( - 'Invalid symbols for enum schema: %r.' % (symbols,)) - - super(EnumSchema, self).__init__( - type=ENUM, - name=name, - namespace=namespace, - names=names, - other_props=other_props, - ) - - self._props['symbols'] = symbols - if doc is not None: - self._props['doc'] = doc - - @property - def symbols(self): - """Returns: the symbols defined in this enum.""" - return self._props['symbols'] - - def to_json(self, names=None): - if names is None: - names = Names() - if self.fullname in names.names: - return self.name_ref(names) - else: - names.names[self.fullname] = self - return names.prune_namespace(self.props) - - def __eq__(self, that): - return self.props == that.props + def __init__( + self, + name, + namespace, + symbols, + names=None, + doc=None, + other_props=None, + ): + """Initializes a new enumeration schema object. + + Args: + name: Simple name of this enumeration. + namespace: Optional namespace. + symbols: Ordered list of symbols defined in this enumeration. + names: + doc: + other_props: + """ + symbols = tuple(symbols) + symbol_set = frozenset(symbols) + if (len(symbol_set) != len(symbols) + or not all(map(lambda symbol: isinstance(symbol, _str), symbols))): + raise AvroException( + 'Invalid symbols for enum schema: %r.' % (symbols,)) + + super(EnumSchema, self).__init__( + data_type=ENUM, + name=name, + namespace=namespace, + names=names, + other_props=other_props, + ) + + self._props['symbols'] = symbols + if doc is not None: + self._props['doc'] = doc + + @property + def symbols(self): + """Returns: the symbols defined in this enum.""" + return self._props['symbols'] + + def to_json(self, names=None): + if names is None: + names = Names() + if self.fullname in names.names: + return self.name_ref(names) + names.names[self.fullname] = self + return names.prune_namespace(self.props) + + def __eq__(self, that): + return self.props == that.props # ------------------------------------------------------------------------------ @@ -736,325 +726,323 @@ def __eq__(self, that): class ArraySchema(Schema): - """Schema of an array.""" - - def __init__(self, items, other_props=None): - """Initializes a new array schema object. - - Args: - items: Avro schema of the array items. - other_props: - """ - super(ArraySchema, self).__init__( - type=ARRAY, - other_props=other_props, - ) - self._items_schema = items - self._props['items'] = items - - @property - def items(self): - """Returns: the schema of the items in this array.""" - return self._items_schema - - def to_json(self, names=None): - if names is None: - names = Names() - to_dump = self.props.copy() - item_schema = self.items - to_dump['items'] = item_schema.to_json(names) - return to_dump - - def __eq__(self, that): - to_cmp = json.loads(str(self)) - return to_cmp == json.loads(str(that)) + """Schema of an array.""" + + def __init__(self, items, other_props=None): + """Initializes a new array schema object. + + Args: + items: Avro schema of the array items. + other_props: + """ + super(ArraySchema, self).__init__( + data_type=ARRAY, + other_props=other_props, + ) + self._items_schema = items + self._props['items'] = items + + @property + def items(self): + """Returns: the schema of the items in this array.""" + return self._items_schema + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = self.props.copy() + item_schema = self.items + to_dump['items'] = item_schema.to_json(names) + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(_str(self)) + return to_cmp == json.loads(_str(that)) # ------------------------------------------------------------------------------ class MapSchema(Schema): - """Schema of a map.""" - - def __init__(self, values, other_props=None): - """Initializes a new map schema object. - - Args: - values: Avro schema of the map values. - other_props: - """ - super(MapSchema, self).__init__( - type=MAP, - other_props=other_props, - ) - self._values_schema = values - self._props['values'] = values - - @property - def values(self): - """Returns: the schema of the values in this map.""" - return self._values_schema - - def to_json(self, names=None): - if names is None: - names = Names() - to_dump = self.props.copy() - to_dump['values'] = self.values.to_json(names) - return to_dump - - def __eq__(self, that): - to_cmp = json.loads(str(self)) - return to_cmp == json.loads(str(that)) + """Schema of a map.""" + + def __init__(self, values, other_props=None): + """Initializes a new map schema object. + + Args: + values: Avro schema of the map values. + other_props: + """ + super(MapSchema, self).__init__( + data_type=MAP, + other_props=other_props, + ) + self._values_schema = values + self._props['values'] = values + + @property + def values(self): + """Returns: the schema of the values in this map.""" + return self._values_schema + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = self.props.copy() + to_dump['values'] = self.values.to_json(names) + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(_str(self)) + return to_cmp == json.loads(_str(that)) # ------------------------------------------------------------------------------ class UnionSchema(Schema): - """Schema of a union.""" - - def __init__(self, schemas): - """Initializes a new union schema object. - - Args: - schemas: Ordered collection of schema branches in the union. - """ - super(UnionSchema, self).__init__(type=UNION) - self._schemas = tuple(schemas) - - # Validate the schema branches: - - # All named schema names are unique: - named_branches = tuple( - filter(lambda schema: schema.type in NAMED_TYPES, self._schemas)) - unique_names = frozenset(map(lambda schema: schema.fullname, named_branches)) - if len(unique_names) != len(named_branches): - raise AvroException( - 'Invalid union branches with duplicate schema name:%s' - % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) - - # Types are unique within unnamed schemas, and union is not allowed: - unnamed_branches = tuple( - filter(lambda schema: schema.type not in NAMED_TYPES, self._schemas)) - unique_types = frozenset(map(lambda schema: schema.type, unnamed_branches)) - if UNION in unique_types: - raise AvroException( - 'Invalid union branches contain other unions:%s' - % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) - if len(unique_types) != len(unnamed_branches): - raise AvroException( - 'Invalid union branches with duplicate type:%s' - % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) - - @property - def schemas(self): - """Returns: the ordered list of schema branches in the union.""" - return self._schemas - - def to_json(self, names=None): - if names is None: - names = Names() - to_dump = [] - for schema in self.schemas: - to_dump.append(schema.to_json(names)) - return to_dump - - def __eq__(self, that): - to_cmp = json.loads(str(self)) - return to_cmp == json.loads(str(that)) + """Schema of a union.""" + + def __init__(self, schemas): + """Initializes a new union schema object. + + Args: + schemas: Ordered collection of schema branches in the union. + """ + super(UnionSchema, self).__init__(data_type=UNION) + self._schemas = tuple(schemas) + + # Validate the schema branches: + + # All named schema names are unique: + named_branches = tuple( + filter(lambda schema: schema.type in NAMED_TYPES, self._schemas)) + unique_names = frozenset(map(lambda schema: schema.fullname, named_branches)) + if len(unique_names) != len(named_branches): + raise AvroException( + 'Invalid union branches with duplicate schema name:%s' + % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) + + # Types are unique within unnamed schemas, and union is not allowed: + unnamed_branches = tuple( + filter(lambda schema: schema.type not in NAMED_TYPES, self._schemas)) + unique_types = frozenset(map(lambda schema: schema.type, unnamed_branches)) + if UNION in unique_types: + raise AvroException( + 'Invalid union branches contain other unions:%s' + % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) + if len(unique_types) != len(unnamed_branches): + raise AvroException( + 'Invalid union branches with duplicate type:%s' + % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) + + @property + def schemas(self): + """Returns: the ordered list of schema branches in the union.""" + return self._schemas + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = [] + for schema in self.schemas: + to_dump.append(schema.to_json(names)) + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(_str(self)) + return to_cmp == json.loads(_str(that)) # ------------------------------------------------------------------------------ class ErrorUnionSchema(UnionSchema): - """Schema representing the declared errors of a protocol message.""" - - def __init__(self, schemas): - """Initializes an error-union schema. - - Args: - schema: collection of error schema. - """ - # TODO: check that string isn't already listed explicitly as an error. - # Prepend "string" to handle system errors - schemas = [PrimitiveSchema(type=STRING)] + list(schemas) - super(ErrorUnionSchema, self).__init__(schemas=schemas) - - def to_json(self, names=None): - if names is None: - names = Names() - to_dump = [] - for schema in self.schemas: - # Don't print the system error schema - if schema.type == STRING: continue - to_dump.append(schema.to_json(names)) - return to_dump + """Schema representing the declared errors of a protocol message.""" + + def __init__(self, schemas): + """Initializes an error-union schema. + + Args: + schema: collection of error schema. + """ + # TODO: check that string isn't already listed explicitly as an error. + # Prepend "string" to handle system errors + schemas = [PrimitiveSchema(data_type=STRING)] + list(schemas) + super(ErrorUnionSchema, self).__init__(schemas=schemas) + + def to_json(self, names=None): + if names is None: + names = Names() + to_dump = [] + for schema in self.schemas: + # Don't print the system error schema + if schema.type == STRING: + continue + to_dump.append(schema.to_json(names)) + return to_dump # ------------------------------------------------------------------------------ class RecordSchema(NamedSchema): - """Schema of a record.""" - - @staticmethod - def _MakeField(index, field_desc, names): - """Builds field schemas from a list of field JSON descriptors. - - Args: - index: 0-based index of the field in the record. - field_desc: JSON descriptors of a record field. - names: Avro schema tracker. - Return: - The field schema. - """ - field_schema = SchemaFromJSONData( - json_data=field_desc['type'], - names=names, - ) - other_props = ( - dict(FilterKeysOut(items=field_desc, keys=FIELD_RESERVED_PROPS))) - return Field( - type=field_schema, - name=field_desc['name'], - index=index, - has_default=('default' in field_desc), - default=field_desc.get('default', _NO_DEFAULT), - order=field_desc.get('order', None), - names=names, - doc=field_desc.get('doc', None), - other_props=other_props, - ) - - @staticmethod - def _MakeFieldList(field_desc_list, names): - """Builds field schemas from a list of field JSON descriptors. - - Guarantees field name unicity. - - Args: - field_desc_list: collection of field JSON descriptors. - names: Avro schema tracker. - Yields - Field schemas. - """ - for index, field_desc in enumerate(field_desc_list): - yield RecordSchema._MakeField(index, field_desc, names) - - @staticmethod - def _MakeFieldMap(fields): - """Builds the field map. - - Guarantees field name unicity. - - Args: - fields: iterable of field schema. - Returns: - A map of field schemas, indexed by name. - """ - field_map = {} - for field in fields: - if field.name in field_map: - raise SchemaParseException( - 'Duplicate record field name %r.' % field.name) - field_map[field.name] = field - return field_map - - def __init__( - self, - name, - namespace, - fields=None, - make_fields=None, - names=None, - record_type=RECORD, - doc=None, - other_props=None - ): - """Initializes a new record schema object. - - Args: - name: Name of the record (absolute or relative). - namespace: Optional namespace the record belongs to, if name is relative. - fields: collection of fields to add to this record. - Exactly one of fields or make_fields must be specified. - make_fields: function creating the fields that belong to the record. - The function signature is: make_fields(names) -> ordered field list. - Exactly one of fields or make_fields must be specified. - names: - record_type: Type of the record: one of RECORD, ERROR or REQUEST. - Protocol requests are not named. - doc: - other_props: - """ - if record_type == REQUEST: - # Protocol requests are not named: - super(NamedSchema, self).__init__( - type=REQUEST, - other_props=other_props, - ) - elif record_type in [RECORD, ERROR]: - # Register this record name in the tracker: - super(RecordSchema, self).__init__( - type=record_type, - name=name, - namespace=namespace, - names=names, - other_props=other_props, - ) - else: - raise SchemaParseException( - 'Invalid record type: %r.' % record_type) - - if record_type in [RECORD, ERROR]: - avro_name = names.GetName(name=name, namespace=namespace) - nested_names = names.NewWithDefaultNamespace(namespace=avro_name.namespace) - elif record_type == REQUEST: - # Protocol request has no name: no need to change default namespace: - nested_names = names - - if fields is None: - fields = make_fields(names=nested_names) - else: - assert (make_fields is None) - self._fields = tuple(fields) - - self._field_map = RecordSchema._MakeFieldMap(self._fields) - - self._props['fields'] = fields - if doc is not None: - self._props['doc'] = doc - - @property - def fields(self): - """Returns: the field schemas, as an ordered tuple.""" - return self._fields - - @property - def field_map(self): - """Returns: a read-only map of the field schemas index by field names.""" - return self._field_map - - def to_json(self, names=None): - if names is None: - names = Names() - # Request records don't have names - if self.type == REQUEST: - return [f.to_json(names) for f in self.fields] - - if self.fullname in names.names: - return self.name_ref(names) - else: - names.names[self.fullname] = self - - to_dump = names.prune_namespace(self.props.copy()) - to_dump['fields'] = [f.to_json(names) for f in self.fields] - return to_dump - - def __eq__(self, that): - to_cmp = json.loads(str(self)) - return to_cmp == json.loads(str(that)) + """Schema of a record.""" + + @staticmethod + def _MakeField(index, field_desc, names): + """Builds field schemas from a list of field JSON descriptors. + + Args: + index: 0-based index of the field in the record. + field_desc: JSON descriptors of a record field. + Return: + The field schema. + """ + field_schema = SchemaFromJSONData( + json_data=field_desc['type'], + names=names, + ) + other_props = ( + dict(FilterKeysOut(items=field_desc, keys=FIELD_RESERVED_PROPS))) + return Field( + data_type=field_schema, + name=field_desc['name'], + index=index, + has_default=('default' in field_desc), + default=field_desc.get('default', _NO_DEFAULT), + order=field_desc.get('order', None), + doc=field_desc.get('doc', None), + other_props=other_props, + ) + + @staticmethod + def MakeFieldList(field_desc_list, names): + """Builds field schemas from a list of field JSON descriptors. + + Guarantees field name unicity. + + Args: + field_desc_list: collection of field JSON descriptors. + names: Avro schema tracker. + Yields + Field schemas. + """ + for index, field_desc in enumerate(field_desc_list): + yield RecordSchema._MakeField(index, field_desc, names) + + @staticmethod + def _MakeFieldMap(fields): + """Builds the field map. + + Guarantees field name unicity. + + Args: + fields: iterable of field schema. + Returns: + A map of field schemas, indexed by name. + """ + field_map = {} + for field in fields: + if field.name in field_map: + raise SchemaParseException( + 'Duplicate record field name %r.' % field.name) + field_map[field.name] = field + return field_map + + def __init__( + self, + name, + namespace, + fields=None, + make_fields=None, + names=None, + record_type=RECORD, + doc=None, + other_props=None + ): + """Initializes a new record schema object. + + Args: + name: Name of the record (absolute or relative). + namespace: Optional namespace the record belongs to, if name is relative. + fields: collection of fields to add to this record. + Exactly one of fields or make_fields must be specified. + make_fields: function creating the fields that belong to the record. + The function signature is: make_fields(names) -> ordered field list. + Exactly one of fields or make_fields must be specified. + names: + record_type: Type of the record: one of RECORD, ERROR or REQUEST. + Protocol requests are not named. + doc: + other_props: + """ + if record_type == REQUEST: + # Protocol requests are not named: + super(RecordSchema, self).__init__( + data_type=REQUEST, + other_props=other_props, + ) + elif record_type in [RECORD, ERROR]: + # Register this record name in the tracker: + super(RecordSchema, self).__init__( + data_type=record_type, + name=name, + namespace=namespace, + names=names, + other_props=other_props, + ) + else: + raise SchemaParseException( + 'Invalid record type: %r.' % record_type) + + if record_type in [RECORD, ERROR]: + avro_name = names.GetName(name=name, namespace=namespace) + nested_names = names.NewWithDefaultNamespace(namespace=avro_name.namespace) + elif record_type == REQUEST: + # Protocol request has no name: no need to change default namespace: + nested_names = names + + if fields is None: + fields = make_fields(names=nested_names) + else: + assert make_fields is None + self._fields = tuple(fields) + + self._field_map = RecordSchema._MakeFieldMap(self._fields) + + self._props['fields'] = fields + if doc is not None: + self._props['doc'] = doc + + @property + def fields(self): + """Returns: the field schemas, as an ordered tuple.""" + return self._fields + + @property + def field_map(self): + """Returns: a read-only map of the field schemas index by field names.""" + return self._field_map + + def to_json(self, names=None): + if names is None: + names = Names() + # Request records don't have names + if self.type == REQUEST: + return [f.to_json(names) for f in self.fields] + + if self.fullname in names.names: + return self.name_ref(names) + names.names[self.fullname] = self + + to_dump = names.prune_namespace(self.props.copy()) + to_dump['fields'] = [f.to_json(names) for f in self.fields] + return to_dump + + def __eq__(self, that): + to_cmp = json.loads(_str(self)) + return to_cmp == json.loads(_str(that)) # ------------------------------------------------------------------------------ @@ -1062,187 +1050,182 @@ def __eq__(self, that): def FilterKeysOut(items, keys): - """Filters a collection of (key, value) items. + """Filters a collection of (key, value) items. - Exclude any item whose key belongs to keys. + Exclude any item whose key belongs to keys. - Args: - items: Dictionary of items to filter the keys out of. - keys: Keys to filter out. - Yields: - Filtered items. - """ - for key, value in items.items(): - if key in keys: continue - yield (key, value) + Args: + items: Dictionary of items to filter the keys out of. + keys: Keys to filter out. + Yields: + Filtered items. + """ + for key, value in items.items(): + if key in keys: + continue + yield key, value # ------------------------------------------------------------------------------ def _SchemaFromJSONString(json_string, names): - if json_string in PRIMITIVE_TYPES: - return PrimitiveSchema(type=json_string) - else: + if json_string in PRIMITIVE_TYPES: + return PrimitiveSchema(data_type=json_string) + # Look for a known named schema: schema = names.GetSchema(name=json_string) if schema is None: - raise SchemaParseException( - 'Unknown named schema %r, known names: %r.' - % (json_string, sorted(names.names))) + raise SchemaParseException( + 'Unknown named schema %r, known names: %r.' + % (json_string, sorted(names.names))) return schema def _SchemaFromJSONArray(json_array, names): - def MakeSchema(desc): - return SchemaFromJSONData(json_data=desc, names=names) - return UnionSchema(map(MakeSchema, json_array)) + def MakeSchema(desc): + return SchemaFromJSONData(json_data=desc, names=names) + return UnionSchema(map(MakeSchema, json_array)) -def _SchemaFromJSONObject(json_object, names): - type = json_object.get('type') - if type is None: - raise SchemaParseException( - 'Avro schema JSON descriptor has no "type" property: %r' % json_object) - - other_props = dict( - FilterKeysOut(items=json_object, keys=SCHEMA_RESERVED_PROPS)) - - if type in PRIMITIVE_TYPES: - # FIXME should not ignore other properties - return PrimitiveSchema(type, other_props=other_props) - - elif type in NAMED_TYPES: - name = json_object.get('name') - namespace = json_object.get('namespace', names.default_namespace) - if type == FIXED: - size = json_object.get('size') - return FixedSchema(name, namespace, size, names, other_props) - elif type == ENUM: - symbols = json_object.get('symbols') - doc = json_object.get('doc') - return EnumSchema(name, namespace, symbols, names, doc, other_props) - - elif type in [RECORD, ERROR]: - field_desc_list = json_object.get('fields', ()) - - def MakeFields(names): - return tuple(RecordSchema._MakeFieldList(field_desc_list, names)) - - return RecordSchema( - name=name, - namespace=namespace, - make_fields=MakeFields, - names=names, - record_type=type, - doc=json_object.get('doc'), - other_props=other_props, - ) - else: - raise Exception('Internal error: unknown type %r.' % type) - - elif type in VALID_TYPES: - # Unnamed, non-primitive Avro type: - if type == ARRAY: - items_desc = json_object.get('items') - if items_desc is None: - raise SchemaParseException( - 'Invalid array schema descriptor with no "items" : %r.' - % json_object) - return ArraySchema( - items=SchemaFromJSONData(items_desc, names), - other_props=other_props, - ) - - elif type == MAP: - values_desc = json_object.get('values') - if values_desc is None: +def _SchemaFromJSONObject(json_object, names): + data_type = json_object.get('type') + if data_type is None: raise SchemaParseException( - 'Invalid map schema descriptor with no "values" : %r.' - % json_object) - return MapSchema( - values=SchemaFromJSONData(values_desc, names=names), - other_props=other_props, - ) - - elif type == ERROR_UNION: - error_desc_list = json_object.get('declared_errors') - assert (error_desc_list is not None) - error_schemas = map( - lambda desc: SchemaFromJSONData(desc, names=names), - error_desc_list) - return ErrorUnionSchema(schemas=error_schemas) - + 'Avro schema JSON descriptor has no "type" property: %r' % json_object) + + other_props = dict( + FilterKeysOut(items=json_object, keys=SCHEMA_RESERVED_PROPS)) + + if data_type in PRIMITIVE_TYPES: + # FIXME should not ignore other properties + result = PrimitiveSchema(data_type, other_props=other_props) + + elif data_type in NAMED_TYPES: + name = json_object.get('name') + namespace = json_object.get('namespace', names.default_namespace) + if data_type == FIXED: + size = json_object.get('size') + result = FixedSchema(name, namespace, size, names, other_props) + elif data_type == ENUM: + symbols = json_object.get('symbols') + doc = json_object.get('doc') + result = EnumSchema(name, namespace, symbols, names, doc, other_props) + + elif data_type in [RECORD, ERROR]: + field_desc_list = json_object.get('fields', ()) + + def MakeFields(names): + return tuple(RecordSchema.MakeFieldList(field_desc_list, names)) + + result = RecordSchema( + name=name, + namespace=namespace, + make_fields=MakeFields, + names=names, + record_type=data_type, + doc=json_object.get('doc'), + other_props=other_props, + ) + else: + raise Exception('Internal error: unknown type %r.' % data_type) + + elif data_type in VALID_TYPES: + # Unnamed, non-primitive Avro type: + + if data_type == ARRAY: + items_desc = json_object.get('items') + if items_desc is None: + raise SchemaParseException( + 'Invalid array schema descriptor with no "items" : %r.' + % json_object) + result = ArraySchema( + items=SchemaFromJSONData(items_desc, names), + other_props=other_props, + ) + + elif data_type == MAP: + values_desc = json_object.get('values') + if values_desc is None: + raise SchemaParseException( + 'Invalid map schema descriptor with no "values" : %r.' + % json_object) + result = MapSchema( + values=SchemaFromJSONData(values_desc, names=names), + other_props=other_props, + ) + + elif data_type == ERROR_UNION: + error_desc_list = json_object.get('declared_errors') + assert error_desc_list is not None + error_schemas = map( + lambda desc: SchemaFromJSONData(desc, names=names), + error_desc_list) + result = ErrorUnionSchema(schemas=error_schemas) + + else: + raise Exception('Internal error: unknown type %r.' % data_type) else: - raise Exception('Internal error: unknown type %r.' % type) - - raise SchemaParseException( - 'Invalid JSON descriptor for an Avro schema: %r' % json_object) + raise SchemaParseException( + 'Invalid JSON descriptor for an Avro schema: %r' % json_object) + return result # Parsers for the JSON data types: _JSONDataParserTypeMap = { - str: _SchemaFromJSONString, - list: _SchemaFromJSONArray, - dict: _SchemaFromJSONObject, + _str: _SchemaFromJSONString, + list: _SchemaFromJSONArray, + dict: _SchemaFromJSONObject, } def SchemaFromJSONData(json_data, names=None): - """Builds an Avro Schema from its JSON descriptor. - - Args: - json_data: JSON data representing the descriptor of the Avro schema. - names: Optional tracker for Avro named schemas. - Returns: - The Avro schema parsed from the JSON descriptor. - Raises: - SchemaParseException: if the descriptor is invalid. - """ - if names is None: - names = Names() + """Builds an Avro Schema from its JSON descriptor. - # Select the appropriate parser based on the JSON data type: - parser = _JSONDataParserTypeMap.get(type(json_data)) - if parser is None: - raise SchemaParseException( - 'Invalid JSON descriptor for an Avro schema: %r.' % json_data) - return parser(json_data, names=names) + Args: + json_data: JSON data representing the descriptor of the Avro schema. + names: Optional tracker for Avro named schemas. + Returns: + The Avro schema parsed from the JSON descriptor. + Raises: + SchemaParseException: if the descriptor is invalid. + """ + if names is None: + names = Names() + + # Select the appropriate parser based on the JSON data type: + parser = _JSONDataParserTypeMap.get(type(json_data)) + if parser is None: + raise SchemaParseException( + 'Invalid JSON descriptor for an Avro schema: %r.' % json_data) + return parser(json_data, names=names) # ------------------------------------------------------------------------------ def parse(json_string): - """Constructs a Schema from its JSON descriptor in text form. - - Args: - json_string: String representation of the JSON descriptor of the schema. - Returns: - The parsed schema. - Raises: - SchemaParseException: on JSON parsing error, - or if the JSON descriptor is invalid. - """ - try: - json_data = json.loads(json_string) - except Exception as exn: - raise SchemaParseException( - 'Error parsing schema from JSON: %r. ' - 'Error message: %r.' - % (json_string, exn)) - - # Initialize the names object - names = Names() - - # construct the Avro Schema object - return SchemaFromJSONData(json_data, names) - - -def Parse(json_string): - """Deprecated implementation of parse.""" - warnings.warn("`Parse` is deprecated in avro 1.9.2. " - "Please use `parse` (lowercase) instead.", - DeprecationWarning) - return parse(json_string) + """Constructs a Schema from its JSON descriptor in text form. + + Args: + json_string: String representation of the JSON descriptor of the schema. + Returns: + The parsed schema. + Raises: + SchemaParseException: on JSON parsing error, + or if the JSON descriptor is invalid. + """ + try: + json_data = json.loads(json_string) + except Exception as exn: + raise SchemaParseException( + 'Error parsing schema from JSON: %r. ' + 'Error message: %r.' + % (json_string, exn)) + + # Initialize the names object + names = Names() + + # construct the Avro Schema object + return SchemaFromJSONData(json_data, names) From 75433fc3a22d352611febcded0ec3ef28cac1ddb Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Wed, 15 Apr 2020 09:33:11 -0700 Subject: [PATCH 05/13] raise StopIteration when there is no more bytes instead of tracking file length. --- .../storage/blob/_shared/avro/avro_io.py | 2 ++ .../storage/blob/_shared/avro/datafile.py | 30 ++----------------- .../tests/_shared/avro/test_avro.py | 19 +++++++++++- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py index e488dc5b6d15..f73613239cf6 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py @@ -148,6 +148,8 @@ def read(self, n): """ assert (n >= 0), n input_bytes = self.reader.read(n) + if n > 0 and not input_bytes: + raise StopIteration assert (len(input_bytes) == n), input_bytes return input_bytes diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py index 6e0581df262b..198d96750a6e 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py @@ -110,8 +110,6 @@ def __init__(self, reader, datum_reader): if self.codec not in VALID_CODECS: raise DataFileException('Unknown codec: %s.' % self.codec) - self._file_length = self._GetInputFileLength() - # get ready to read self._block_count = 0 self.datum_reader.writer_schema = ( @@ -153,11 +151,6 @@ def sync_marker(self): def meta(self): return self._meta - @property - def file_length(self): - """Length of the input file, in bytes.""" - return self._file_length - # read/write properties @property def block_count(self): @@ -184,23 +177,6 @@ def SetMeta(self, key, value): value = value.encode('utf-8') self._meta[key] = value - def _GetInputFileLength(self): - """Reports the length of the input file, in bytes. - - Leaves the current position unmodified. - - Returns: - The length of the input file, in bytes. - """ - current_pos = self.reader.tell() - self.reader.seek(0, 2) - file_length = self.reader.tell() - self.reader.seek(current_pos) - return file_length - - def is_EOF(self): - return self.reader.tell() == self.file_length - def _read_header(self): # seek to the beginning of the file to get magic block self.reader.seek(0, 0) @@ -244,6 +220,8 @@ def _skip_sync(self): return True. Otherwise, seek back to where we started and return False. """ proposed_sync_marker = self.reader.read(SYNC_SIZE) + if SYNC_SIZE > 0 and not proposed_sync_marker: + raise StopIteration if proposed_sync_marker != self.sync_marker: self.reader.seek(-SYNC_SIZE, 1) return False @@ -252,11 +230,7 @@ def _skip_sync(self): def __next__(self): """Return the next datum in the file.""" if self.block_count == 0: - if self.is_EOF(): - raise StopIteration if self._skip_sync(): - if self.is_EOF(): - raise StopIteration self._read_block_header() else: self._read_block_header() diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py index 980a6bc651ea..4ef89f752eb9 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py +++ b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py @@ -9,7 +9,7 @@ import inspect import os import unittest -from io import open +from io import BytesIO, open from azure.storage.blob._shared.avro.datafile import DataFileReader from azure.storage.blob._shared.avro.avro_io import DatumReader @@ -84,6 +84,23 @@ def test_reader(self): correct, len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE)) + def test_reader_with_bytes_io(self): + correct = 0 + nitems = 10 + for iexample, (writer_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE): + for codec in CODECS_TO_VALIDATE: + file_path = os.path.join(AvroReaderTests._samples_dir_root, 'test_' + codec + '_' + str(iexample) + '.avro') + with open(file_path, 'rb') as reader: + data = BytesIO(reader.read()) + datum_reader = DatumReader() + with DataFileReader(data, datum_reader) as dfr: + round_trip_data = list(dfr) + if ([datum] * nitems) == round_trip_data: + correct += 1 + self.assertEqual( + correct, + len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE)) + def test_change_feed(self): file_path = os.path.join(AvroReaderTests._samples_dir_root, 'changeFeed.avro') with open(file_path, 'rb') as reader: From a93d5018b8953d5d17d78ace1916d333203309fd Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Thu, 16 Apr 2020 10:42:32 -0700 Subject: [PATCH 06/13] async avro parser --- .../storage/blob/_shared/avro/avro_io.py | 60 -- .../blob/_shared/avro/avro_io_async.py | 595 ++++++++++++++++++ .../storage/blob/_shared/avro/datafile.py | 10 - .../blob/_shared/avro/datafile_async.py | 191 ++++++ .../tests/_shared/avro/test_avro_async.py | 91 +++ 5 files changed, 877 insertions(+), 70 deletions(-) create mode 100644 sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py create mode 100644 sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py create mode 100644 sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py index f73613239cf6..d16313954593 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py @@ -42,32 +42,14 @@ # ------------------------------------------------------------------------------ # Constants - -INT_MIN_VALUE = -(1 << 31) -INT_MAX_VALUE = (1 << 31) - 1 -LONG_MIN_VALUE = -(1 << 63) -LONG_MAX_VALUE = (1 << 63) - 1 - -STRUCT_INT = struct.Struct('!I') # big-endian unsigned int STRUCT_FLOAT = struct.Struct('I') # big-endian unsigned int - # ------------------------------------------------------------------------------ # Exceptions -class AvroTypeException(schema.AvroException): - """Raised when datum is not an example of schema.""" - - def __init__(self, expected_schema, datum): - pretty_expected = json.dumps(json.loads(str(expected_schema)), indent=2) - fail_msg = "The datum %s is not an example of the schema %s" \ - % (datum, pretty_expected) - schema.AvroException.__init__(self, fail_msg) - - class SchemaResolutionException(schema.AvroException): def __init__(self, fail_msg, writer_schema=None, reader_schema=None): pretty_writers = json.dumps(json.loads(str(writer_schema)), indent=2) @@ -78,48 +60,6 @@ def __init__(self, fail_msg, writer_schema=None, reader_schema=None): fail_msg += "\nReader's Schema: %s" % pretty_readers schema.AvroException.__init__(self, fail_msg) - -# ------------------------------------------------------------------------------ -# Validate - -_valid = { - 'null': lambda s, d: d is None, - 'boolean': lambda s, d: isinstance(d, bool), - 'string': lambda s, d: isinstance(d, str), - 'bytes': lambda s, d: isinstance(d, bytes), - 'int': lambda s, d: isinstance(d, int) and (INT_MIN_VALUE <= d <= INT_MAX_VALUE), - 'long': lambda s, d: isinstance(d, int) and (LONG_MIN_VALUE <= d <= LONG_MAX_VALUE), - 'float': lambda s, d: isinstance(d, (int, float)), - 'fixed': lambda s, d: isinstance(d, bytes) and len(d) == s.size, - 'enum': lambda s, d: d in s.symbols, - 'array': lambda s, d: isinstance(d, list) and all(Validate(s.items, item) for item in d), - 'map': lambda s, d: (isinstance(d, dict) and all(isinstance(key, str) for key in d) - and all(Validate(s.values, value) for value in d.values())), - 'union': lambda s, d: any(Validate(branch, d) for branch in s.schemas), - 'record': lambda s, d: (isinstance(d, dict) - and all(Validate(f.type, d.get(f.name)) for f in s.fields) - and {f.name for f in s.fields}.issuperset(d.keys())) -} -_valid['double'] = _valid['float'] -_valid['error_union'] = _valid['union'] -_valid['error'] = _valid['request'] = _valid['record'] - - -def Validate(expected_schema, datum): - """Determines if a python datum is an instance of a schema. - - Args: - expected_schema: Schema to validate against. - datum: Datum to validate. - Returns: - True if the datum is an instance of the schema. - """ - try: - return _valid[expected_schema.type](expected_schema, datum) - except KeyError: - raise AvroTypeException(expected_schema, datum) - - # ------------------------------------------------------------------------------ # Decoder/Encoder diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py new file mode 100644 index 000000000000..07ffe160b823 --- /dev/null +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py @@ -0,0 +1,595 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +"""Input/output utilities. + +Includes: + - i/o-specific constants + - i/o-specific exceptions + - schema validation + - leaf value encoding and decoding + - datum reader/writer stuff (?) + +Also includes a generic representation for data, which uses the +following mapping: + - Schema records are implemented as dict. + - Schema arrays are implemented as list. + - Schema maps are implemented as dict. + - Schema strings are implemented as unicode. + - Schema bytes are implemented as str. + - Schema ints are implemented as int. + - Schema longs are implemented as long. + - Schema floats are implemented as float. + - Schema doubles are implemented as float. + - Schema booleans are implemented as bool. +""" + +import binascii +import logging +import sys + +from ..avro import schema + +from .avro_io import STRUCT_FLOAT, STRUCT_DOUBLE, STRUCT_CRC32, SchemaResolutionException + +PY3 = sys.version_info[0] == 3 + +logger = logging.getLogger(__name__) + +# ------------------------------------------------------------------------------ +# Decoder/Encoder + + +class AsyncBinaryDecoder(object): + """Read leaf values.""" + + def __init__(self, reader): + """ + reader is a Python object on which we can call read, seek, and tell. + """ + self._reader = reader + + @property + def reader(self): + """Reports the reader used by this decoder.""" + return self._reader + + async def read(self, n): + """Read n bytes. + + Args: + n: Number of bytes to read. + Returns: + The next n bytes from the input. + """ + assert (n >= 0), n + input_bytes = await self.reader.read(n) + if n > 0 and not input_bytes: + raise StopAsyncIteration + assert (len(input_bytes) == n), input_bytes + return input_bytes + + @staticmethod + def read_null(): + """ + null is written as zero bytes + """ + return None + + async def read_boolean(self): + """ + a boolean is written as a single byte + whose value is either 0 (false) or 1 (true). + """ + return ord(await self.read(1)) == 1 + + async def read_int(self): + """ + int and long values are written using variable-length, zig-zag coding. + """ + return await self.read_long() + + async def read_long(self): + """ + int and long values are written using variable-length, zig-zag coding. + """ + b = ord(await self.read(1)) + n = b & 0x7F + shift = 7 + while (b & 0x80) != 0: + b = ord(await self.read(1)) + n |= (b & 0x7F) << shift + shift += 7 + datum = (n >> 1) ^ -(n & 1) + return datum + + async def read_float(self): + """ + A float is written as 4 bytes. + The float is converted into a 32-bit integer using a method equivalent to + Java's floatToIntBits and then encoded in little-endian format. + """ + return STRUCT_FLOAT.unpack(await self.read(4))[0] + + async def read_double(self): + """ + A double is written as 8 bytes. + The double is converted into a 64-bit integer using a method equivalent to + Java's doubleToLongBits and then encoded in little-endian format. + """ + return STRUCT_DOUBLE.unpack(await self.read(8))[0] + + async def read_bytes(self): + """ + Bytes are encoded as a long followed by that many bytes of data. + """ + nbytes = await self.read_long() + assert (nbytes >= 0), nbytes + return await self.read(nbytes) + + async def read_utf8(self): + """ + A string is encoded as a long followed by + that many bytes of UTF-8 encoded character data. + """ + input_bytes = await self.read_bytes() + if PY3: + try: + return input_bytes.decode('utf-8') + except UnicodeDecodeError as exn: + logger.error('Invalid UTF-8 input bytes: %r', input_bytes) + raise exn + else: + # PY2 + return unicode(input_bytes, "utf-8") # pylint: disable=undefined-variable + + async def check_crc32(self, input_bytes): + checksum = STRUCT_CRC32.unpack(await self.read(4))[0] + if binascii.crc32(input_bytes) & 0xffffffff != checksum: + raise schema.AvroException("Checksum failure") + + def skip_null(self): + pass + + async def skip_boolean(self): + await self.skip(1) + + async def skip_int(self): + await self.skip_long() + + async def skip_long(self): + b = ord(await self.read(1)) + while (b & 0x80) != 0: + b = ord(await self.read(1)) + + async def skip_float(self): + await self.skip(4) + + async def skip_double(self): + await self.skip(8) + + async def skip_bytes(self): + await self.skip(await self.read_long()) + + async def skip_utf8(self): + await self.skip_bytes() + + async def skip(self, n): + await self.reader.seek(await self.reader.tell() + n) + + +# ------------------------------------------------------------------------------ +# DatumReader/Writer + + +class AsyncDatumReader(object): + """Deserialize Avro-encoded data into a Python data structure.""" + + @staticmethod + def check_props(schema_one, schema_two, prop_list): + for prop in prop_list: + if getattr(schema_one, prop) != getattr(schema_two, prop): + return False + return True + + @staticmethod + def match_schemas(writer_schema, reader_schema): + w_type = writer_schema.type + r_type = reader_schema.type + if 'union' in [w_type, r_type] or 'error_union' in [w_type, r_type]: + result = True + elif (w_type in schema.PRIMITIVE_TYPES and r_type in schema.PRIMITIVE_TYPES + and w_type == r_type): + result = True + elif (w_type == r_type == 'record' and + AsyncDatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + result = True + elif (w_type == r_type == 'error' and + AsyncDatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + result = True + elif w_type == r_type == 'request': + return True + elif (w_type == r_type == 'fixed' and + AsyncDatumReader.check_props(writer_schema, reader_schema, + ['fullname', 'size'])): + result = True + elif (w_type == r_type == 'enum' and + AsyncDatumReader.check_props(writer_schema, reader_schema, + ['fullname'])): + result = True + elif (w_type == r_type == 'map' and + AsyncDatumReader.check_props(writer_schema.values, + reader_schema.values, ['type'])): + result = True + elif (w_type == r_type == 'array' and + AsyncDatumReader.check_props(writer_schema.items, + reader_schema.items, ['type'])): + result = True + + # Handle schema promotion + elif w_type == 'int' and r_type in ['long', 'float', 'double']: + result = True + elif w_type == 'long' and r_type in ['float', 'double']: + result = True + elif w_type == 'float' and r_type == 'double': + result = True + else: + result = False + return result + + def __init__(self, writer_schema=None, reader_schema=None): + """ + As defined in the Avro specification, we call the schema encoded + in the data the "writer's schema", and the schema expected by the + reader the "reader's schema". + """ + self._writer_schema = writer_schema + self._reader_schema = reader_schema + + # read/write properties + def set_writer_schema(self, writer_schema): + self._writer_schema = writer_schema + + writer_schema = property(lambda self: self._writer_schema, + set_writer_schema) + + def set_reader_schema(self, reader_schema): + self._reader_schema = reader_schema + + reader_schema = property(lambda self: self._reader_schema, + set_reader_schema) + + async def read(self, decoder): + if self.reader_schema is None: + self.reader_schema = self.writer_schema + return await self.read_data(self.writer_schema, self.reader_schema, decoder) + + async def read_data(self, writer_schema, reader_schema, decoder): + # schema matching + if not AsyncDatumReader.match_schemas(writer_schema, reader_schema): + fail_msg = 'Schemas do not match.' + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + # schema resolution: reader's schema is a union, writer's schema is not + if (writer_schema.type not in ['union', 'error_union'] + and reader_schema.type in ['union', 'error_union']): + for s in reader_schema.schemas: + if AsyncDatumReader.match_schemas(writer_schema, s): + return await self.read_data(writer_schema, s, decoder) + fail_msg = 'Schemas do not match.' + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + # function dispatch for reading data based on type of writer's schema + if writer_schema.type == 'null': + result = decoder.read_null() + elif writer_schema.type == 'boolean': + result = await decoder.read_boolean() + elif writer_schema.type == 'string': + result = await decoder.read_utf8() + elif writer_schema.type == 'int': + result = await decoder.read_int() + elif writer_schema.type == 'long': + result = await decoder.read_long() + elif writer_schema.type == 'float': + result = await decoder.read_float() + elif writer_schema.type == 'double': + result = await decoder.read_double() + elif writer_schema.type == 'bytes': + result = await decoder.read_bytes() + elif writer_schema.type == 'fixed': + result = await self.read_fixed(writer_schema, decoder) + elif writer_schema.type == 'enum': + result = await self.read_enum(writer_schema, reader_schema, decoder) + elif writer_schema.type == 'array': + result = await self.read_array(writer_schema, reader_schema, decoder) + elif writer_schema.type == 'map': + result = await self.read_map(writer_schema, reader_schema, decoder) + elif writer_schema.type in ['union', 'error_union']: + result = await self.read_union(writer_schema, reader_schema, decoder) + elif writer_schema.type in ['record', 'error', 'request']: + result = await self.read_record(writer_schema, reader_schema, decoder) + else: + fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type + raise schema.AvroException(fail_msg) + return result + + async def skip_data(self, writer_schema, decoder): + if writer_schema.type == 'null': + result = decoder.skip_null() + elif writer_schema.type == 'boolean': + result = await decoder.skip_boolean() + elif writer_schema.type == 'string': + result = await decoder.skip_utf8() + elif writer_schema.type == 'int': + result = await decoder.skip_int() + elif writer_schema.type == 'long': + result = await decoder.skip_long() + elif writer_schema.type == 'float': + result = await decoder.skip_float() + elif writer_schema.type == 'double': + result = await decoder.skip_double() + elif writer_schema.type == 'bytes': + result = await decoder.skip_bytes() + elif writer_schema.type == 'fixed': + result = await self.skip_fixed(writer_schema, decoder) + elif writer_schema.type == 'enum': + result = await self.skip_enum(decoder) + elif writer_schema.type == 'array': + await self.skip_array(writer_schema, decoder) + result = None + elif writer_schema.type == 'map': + await self.skip_map(writer_schema, decoder) + result = None + elif writer_schema.type in ['union', 'error_union']: + result = await self.skip_union(writer_schema, decoder) + elif writer_schema.type in ['record', 'error', 'request']: + await self.skip_record(writer_schema, decoder) + result = None + else: + fail_msg = "Unknown schema type: %s" % writer_schema.type + raise schema.AvroException(fail_msg) + return result + + @staticmethod + async def read_fixed(writer_schema, decoder): + """ + Fixed instances are encoded using the number of bytes declared + in the schema. + """ + return await decoder.read(writer_schema.size) + + @staticmethod + async def skip_fixed(writer_schema, decoder): + return await decoder.skip(writer_schema.size) + + @staticmethod + async def read_enum(writer_schema, reader_schema, decoder): + """ + An enum is encoded by a int, representing the zero-based position + of the symbol in the schema. + """ + # read data + index_of_symbol = await decoder.read_int() + if index_of_symbol >= len(writer_schema.symbols): + fail_msg = "Can't access enum index %d for enum with %d symbols" \ + % (index_of_symbol, len(writer_schema.symbols)) + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + read_symbol = writer_schema.symbols[index_of_symbol] + + # schema resolution + if read_symbol not in reader_schema.symbols: + fail_msg = "Symbol %s not present in Reader's Schema" % read_symbol + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + + return read_symbol + + @staticmethod + async def skip_enum(decoder): + return await decoder.skip_int() + + async def read_array(self, writer_schema, reader_schema, decoder): + """ + Arrays are encoded as a series of blocks. + + Each block consists of a long count value, + followed by that many array items. + A block with count zero indicates the end of the array. + Each item is encoded per the array's item schema. + + If a block's count is negative, + then the count is followed immediately by a long block size, + indicating the number of bytes in the block. + The actual count in this case + is the absolute value of the count written. + """ + read_items = [] + block_count = await decoder.read_long() + while block_count != 0: + if block_count < 0: + block_count = -block_count + await decoder.read_long() + for _ in range(block_count): + read_items.append(await self.read_data(writer_schema.items, + reader_schema.items, decoder)) + block_count = await decoder.read_long() + return read_items + + async def skip_array(self, writer_schema, decoder): + block_count = await decoder.read_long() + while block_count != 0: + if block_count < 0: + block_size = await decoder.read_long() + await decoder.skip(block_size) + else: + for _ in range(block_count): + await self.skip_data(writer_schema.items, decoder) + block_count = await decoder.read_long() + + async def read_map(self, writer_schema, reader_schema, decoder): + """ + Maps are encoded as a series of blocks. + + Each block consists of a long count value, + followed by that many key/value pairs. + A block with count zero indicates the end of the map. + Each item is encoded per the map's value schema. + + If a block's count is negative, + then the count is followed immediately by a long block size, + indicating the number of bytes in the block. + The actual count in this case + is the absolute value of the count written. + """ + read_items = {} + block_count = await decoder.read_long() + while block_count != 0: + if block_count < 0: + block_count = -block_count + await decoder.read_long() + for _ in range(block_count): + key = await decoder.read_utf8() + read_items[key] = await self.read_data(writer_schema.values, + reader_schema.values, decoder) + block_count = await decoder.read_long() + return read_items + + async def skip_map(self, writer_schema, decoder): + block_count = await decoder.read_long() + while block_count != 0: + if block_count < 0: + block_size = await decoder.read_long() + await decoder.skip(block_size) + else: + for _ in range(block_count): + await decoder.skip_utf8() + await self.skip_data(writer_schema.values, decoder) + block_count = await decoder.read_long() + + async def read_union(self, writer_schema, reader_schema, decoder): + """ + A union is encoded by first writing a long value indicating + the zero-based position within the union of the schema of its value. + The value is then encoded per the indicated schema within the union. + """ + # schema resolution + index_of_schema = int(await decoder.read_long()) + if index_of_schema >= len(writer_schema.schemas): + fail_msg = "Can't access branch index %d for union with %d branches" \ + % (index_of_schema, len(writer_schema.schemas)) + raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + selected_writer_schema = writer_schema.schemas[index_of_schema] + + # read data + return await self.read_data(selected_writer_schema, reader_schema, decoder) + + async def skip_union(self, writer_schema, decoder): + index_of_schema = int(await decoder.read_long()) + if index_of_schema >= len(writer_schema.schemas): + fail_msg = "Can't access branch index %d for union with %d branches" \ + % (index_of_schema, len(writer_schema.schemas)) + raise SchemaResolutionException(fail_msg, writer_schema) + return await self.skip_data(writer_schema.schemas[index_of_schema], decoder) + + async def read_record(self, writer_schema, reader_schema, decoder): + """ + A record is encoded by encoding the values of its fields + in the order that they are declared. In other words, a record + is encoded as just the concatenation of the encodings of its fields. + Field values are encoded per their schema. + + Schema Resolution: + * the ordering of fields may be different: fields are matched by name. + * schemas for fields with the same name in both records are resolved + recursively. + * if the writer's record contains a field with a name not present in the + reader's record, the writer's value for that field is ignored. + * if the reader's record schema has a field that contains a default value, + and writer's schema does not have a field with the same name, then the + reader should use the default value from its field. + * if the reader's record schema has a field with no default value, and + writer's schema does not have a field with the same name, then the + field's value is unset. + """ + # schema resolution + readers_fields_dict = reader_schema.field_map + read_record = {} + for field in writer_schema.fields: + readers_field = readers_fields_dict.get(field.name) + if readers_field is not None: + field_val = await self.read_data(field.type, readers_field.type, decoder) + read_record[field.name] = field_val + else: + await self.skip_data(field.type, decoder) + + # fill in default values + if len(readers_fields_dict) > len(read_record): + writers_fields_dict = writer_schema.field_map + for field_name, field in readers_fields_dict.items(): + if field_name not in writers_fields_dict: + if field.has_default: + field_val = self._read_default_value(field.type, field.default) + read_record[field.name] = field_val + else: + fail_msg = 'No default value for field %s' % field_name + raise SchemaResolutionException(fail_msg, writer_schema, + reader_schema) + return read_record + + async def skip_record(self, writer_schema, decoder): + for field in writer_schema.fields: + await self.skip_data(field.type, decoder) + + def _read_default_value(self, field_schema, default_value): + """ + Basically a JSON Decoder? + """ + if field_schema.type == 'null': + result = None + elif field_schema.type == 'boolean': + result = bool(default_value) + elif field_schema.type == 'int': + result = int(default_value) + elif field_schema.type == 'long': + result = int(default_value) + elif field_schema.type in ['float', 'double']: + result = float(default_value) + elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']: + result = default_value + elif field_schema.type == 'array': + read_array = [] + for json_val in default_value: + item_val = self._read_default_value(field_schema.items, json_val) + read_array.append(item_val) + result = read_array + elif field_schema.type == 'map': + read_map = {} + for key, json_val in default_value.items(): + map_val = self._read_default_value(field_schema.values, json_val) + read_map[key] = map_val + result = read_map + elif field_schema.type in ['union', 'error_union']: + result = self._read_default_value(field_schema.schemas[0], default_value) + elif field_schema.type == 'record': + read_record = {} + for field in field_schema.fields: + json_val = default_value.get(field.name) + if json_val is None: + json_val = field.default + field_val = self._read_default_value(field.type, json_val) + read_record[field.name] = field_val + result = read_record + else: + fail_msg = 'Unknown type: %s' % field_schema.type + raise schema.AvroException(fail_msg) + return result + + +# ------------------------------------------------------------------------------ + +if __name__ == '__main__': + raise Exception('Not a standalone module') diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py index 198d96750a6e..d88ad331289e 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py @@ -34,10 +34,6 @@ # Size of the synchronization marker, in number of bytes: SYNC_SIZE = 16 -# Interval between synchronization markers, in number of bytes: -# TODO: make configurable -SYNC_INTERVAL = 1000 * SYNC_SIZE - # Schema of the container header: META_SCHEMA = schema.parse(""" { @@ -61,12 +57,6 @@ # Codecs supported by container files: VALID_CODECS = frozenset(['null', 'deflate']) -# Not used yet -VALID_ENCODINGS = frozenset(['binary']) - -# Metadata key associated to the codec: -CODEC_KEY = "avro.codec" - # Metadata key associated to the schema: SCHEMA_KEY = "avro.schema" diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py new file mode 100644 index 000000000000..2fc33b139533 --- /dev/null +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py @@ -0,0 +1,191 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +"""Read/Write Avro File Object Containers.""" + +import logging +import sys + +from ..avro import avro_io_async +from ..avro import schema +from .datafile import DataFileException +from .datafile import MAGIC, SYNC_SIZE, META_SCHEMA, SCHEMA_KEY + + +PY3 = sys.version_info[0] == 3 + +logger = logging.getLogger(__name__) + +# ------------------------------------------------------------------------------ +# Constants + +# Codecs supported by container files: +VALID_CODECS = frozenset(['null']) + + +class AsyncDataFileReader(object): + """Read files written by DataFileWriter.""" + + # TODO: allow user to specify expected schema? + # TODO: allow user to specify the encoder + def __init__(self, reader, datum_reader): + """Initializes a new data file reader. + + Args: + reader: Open file to read from. + datum_reader: Avro datum reader. + """ + self._reader = reader + self._raw_decoder = avro_io_async.AsyncBinaryDecoder(reader) + self._datum_decoder = None # Maybe reset at every block. + self._datum_reader = datum_reader + self.codec = "null" + self._block_count = 0 + self._meta = None + self._sync_marker = None + + async def init(self): + # read the header: magic, meta, sync + await self._read_header() + + # ensure codec is valid + avro_codec_raw = self.GetMeta('avro.codec') + if avro_codec_raw is None: + self.codec = "null" + else: + self.codec = avro_codec_raw.decode('utf-8') + if self.codec not in VALID_CODECS: + raise DataFileException('Unknown codec: %s.' % self.codec) + + # get ready to read + self._block_count = 0 + self.datum_reader.writer_schema = ( + schema.parse(self.GetMeta(SCHEMA_KEY).decode('utf-8'))) + return self + + async def __aenter__(self): + return self + + async def __aexit__(self, data_type, value, traceback): + # Perform a close if there's no exception + if data_type is None: + self.close() + + def __aiter__(self): + return self + + # read-only properties + @property + def reader(self): + return self._reader + + @property + def raw_decoder(self): + return self._raw_decoder + + @property + def datum_decoder(self): + return self._datum_decoder + + @property + def datum_reader(self): + return self._datum_reader + + @property + def sync_marker(self): + return self._sync_marker + + @property + def meta(self): + return self._meta + + # read/write properties + @property + def block_count(self): + return self._block_count + + def GetMeta(self, key): + """Reports the value of a given metadata key. + + Args: + key: Metadata key (string) to report the value of. + Returns: + Value associated to the metadata key, as bytes. + """ + return self._meta.get(key) + + def SetMeta(self, key, value): + """Sets a metadata. + + Args: + key: Metadata key (string) to set. + value: Metadata value to set, as bytes. + """ + if isinstance(value, str): + value = value.encode('utf-8') + self._meta[key] = value + + async def _read_header(self): + # seek to the beginning of the file to get magic block + await self.reader.seek(0, 0) + + # read header into a dict + header = await self.datum_reader.read_data( + META_SCHEMA, META_SCHEMA, self.raw_decoder) + + # check magic number + if header.get('magic') != MAGIC: + fail_msg = "Not an Avro data file: %s doesn't match %s." \ + % (header.get('magic'), MAGIC) + raise schema.AvroException(fail_msg) + + # set metadata + self._meta = header['meta'] + + # set sync marker + self._sync_marker = header['sync'] + + async def _read_block_header(self): + self._block_count = await self.raw_decoder.read_long() + if self.codec == "null": + # Skip a long; we don't need to use the length. + await self.raw_decoder.skip_long() + self._datum_decoder = self._raw_decoder + else: + raise DataFileException("Unknown codec: %r" % self.codec) + + async def _skip_sync(self): + """ + Read the length of the sync marker; if it matches the sync marker, + return True. Otherwise, seek back to where we started and return False. + """ + proposed_sync_marker = await self.reader.read(SYNC_SIZE) + if SYNC_SIZE > 0 and not proposed_sync_marker: + raise StopAsyncIteration + if proposed_sync_marker != self.sync_marker: + await self.reader.seek(-SYNC_SIZE, 1) + return False + return True + + async def __anext__(self): + """Return the next datum in the file.""" + if self.block_count == 0: + if await self._skip_sync(): + await self._read_block_header() + else: + await self._read_block_header() + + datum = await self.datum_reader.read(self.datum_decoder) + self._block_count -= 1 + return datum + + def close(self): + """Close this reader.""" + self.reader.close() + + +if __name__ == '__main__': + raise Exception('Not a standalone module') diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py new file mode 100644 index 000000000000..5b50914fad3f --- /dev/null +++ b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py @@ -0,0 +1,91 @@ + +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import inspect +import os +import pytest +import unittest +from azure.storage.blob._shared.avro.avro_io_async import AsyncDatumReader +from azure.storage.blob._shared.avro.datafile_async import AsyncDataFileReader + +from .test_avro import SCHEMAS_TO_VALIDATE + +CODECS_TO_VALIDATE = ['null'] + + +class AsyncBufferedReaderWrapper: + def __init__(self, reader): + self._reader = reader + + async def seek(self, offset, whence=0): + self._reader.seek(offset, whence) + + async def read(self, size=None): + return self._reader.read(size) + + def close(self): + self._reader.close() + + +class AvroReaderTestsAsync(unittest.TestCase): + @classmethod + def setUpClass(cls): + test_file_path = inspect.getfile(cls) + cls._samples_dir_root = os.path.join(os.path.dirname(test_file_path), 'samples') + + @pytest.mark.asyncio + async def test_reader(self): + correct = 0 + nitems = 10 + for iexample, (writer_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE): + for codec in CODECS_TO_VALIDATE: + file_path = os.path.join(AvroReaderTestsAsync._samples_dir_root, 'test_' + codec + '_' + str(iexample) + '.avro') + with open(file_path, 'rb') as reader: + datum_reader = AsyncDatumReader() + async_reader = AsyncBufferedReaderWrapper(reader) + async with await AsyncDataFileReader(async_reader, datum_reader).init() as dfr: + round_trip_data = [x async for x in dfr] + if ([datum] * nitems) == round_trip_data: + correct += 1 + self.assertEqual( + correct, + len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE)) + + @pytest.mark.asyncio + async def test_change_feed(self): + file_path = os.path.join(AvroReaderTestsAsync._samples_dir_root, 'changeFeed.avro') + with open(file_path, 'rb') as reader: + datum_reader = AsyncDatumReader() + async_reader = AsyncBufferedReaderWrapper(reader) + async with await AsyncDataFileReader(async_reader, datum_reader).init() as dfr: + data = [x async for x in dfr] + self.assertEqual(1, len(data)) + expectedRecord = { + 'data': { + 'api': 'PutBlob', + 'blobPropertiesUpdated': None, + 'blobType': 'BlockBlob', + 'clientRequestId': '75b6c460-fcd0-11e9-87e2-85def057dae9', + 'contentLength': 12, + 'contentType': 'text/plain', + 'etag': '0x8D75EF45A3B8617', + 'previousInfo': None, + 'requestId': 'bb219c8e-401e-0028-1fdd-90f393000000', + 'sequencer': '000000000000000000000000000017140000000000000fcc', + 'snapshot': None, + 'storageDiagnostics': {'bid': 'd3053fa1-a006-0042-00dd-902bbb000000', + 'seq': '(5908,134,4044,0)', + 'sid': '5aaf98bf-f1d8-dd76-2dd2-9b60c689538d'}, + 'url': ''}, + 'eventTime': '2019-11-01T17:53:07.5106080Z', + 'eventType': 'BlobCreated', + 'id': 'bb219c8e-401e-0028-1fdd-90f393069ae4', + 'schemaVersion': 3, + 'subject': '/blobServices/default/containers/test/blobs/sdf.txt', + 'topic': '/subscriptions/ba45b233-e2ef-4169-8808-49eb0d8eba0d/resourceGroups/XClient/providers/Microsoft.Storage/storageAccounts/seanchangefeedstage'} + self.assertEqual(expectedRecord, data[0]) \ No newline at end of file From a1524ed5fe9fce751eb524ee2f448d3646240010 Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Thu, 16 Apr 2020 14:42:00 -0700 Subject: [PATCH 07/13] fix syntax for Python 3.5 --- .../tests/_shared/avro/test_avro_async.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py index 5b50914fad3f..eeba0850bcaf 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py +++ b/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py @@ -49,7 +49,9 @@ async def test_reader(self): datum_reader = AsyncDatumReader() async_reader = AsyncBufferedReaderWrapper(reader) async with await AsyncDataFileReader(async_reader, datum_reader).init() as dfr: - round_trip_data = [x async for x in dfr] + round_trip_data = [] + async for x in dfr: + round_trip_data.append(x) if ([datum] * nitems) == round_trip_data: correct += 1 self.assertEqual( @@ -63,7 +65,9 @@ async def test_change_feed(self): datum_reader = AsyncDatumReader() async_reader = AsyncBufferedReaderWrapper(reader) async with await AsyncDataFileReader(async_reader, datum_reader).init() as dfr: - data = [x async for x in dfr] + data = [] + async for x in dfr: + data.append(x) self.assertEqual(1, len(data)) expectedRecord = { 'data': { From 2074fa748f83ba349a7fc036f6929c0a9a2ab834 Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Wed, 22 Apr 2020 09:32:07 -0700 Subject: [PATCH 08/13] get rid of 'readers_schema' as we only honor schema that has been written to file ('writer_schema'). --- .../storage/blob/_shared/avro/avro_io.py | 151 +++--------------- .../blob/_shared/avro/avro_io_async.py | 143 +++-------------- .../storage/blob/_shared/avro/datafile.py | 3 +- .../blob/_shared/avro/datafile_async.py | 3 +- 4 files changed, 44 insertions(+), 256 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py index d16313954593..e7aff7a17424 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py @@ -51,13 +51,10 @@ class SchemaResolutionException(schema.AvroException): - def __init__(self, fail_msg, writer_schema=None, reader_schema=None): + def __init__(self, fail_msg, writer_schema=None): pretty_writers = json.dumps(json.loads(str(writer_schema)), indent=2) - pretty_readers = json.dumps(json.loads(str(reader_schema)), indent=2) if writer_schema: fail_msg += "\nWriter's Schema: %s" % pretty_writers - if reader_schema: - fail_msg += "\nReader's Schema: %s" % pretty_readers schema.AvroException.__init__(self, fail_msg) # ------------------------------------------------------------------------------ @@ -209,68 +206,12 @@ def skip(self, n): class DatumReader(object): """Deserialize Avro-encoded data into a Python data structure.""" - @staticmethod - def check_props(schema_one, schema_two, prop_list): - for prop in prop_list: - if getattr(schema_one, prop) != getattr(schema_two, prop): - return False - return True - - @staticmethod - def match_schemas(writer_schema, reader_schema): - w_type = writer_schema.type - r_type = reader_schema.type - if 'union' in [w_type, r_type] or 'error_union' in [w_type, r_type]: - result = True - elif (w_type in schema.PRIMITIVE_TYPES and r_type in schema.PRIMITIVE_TYPES - and w_type == r_type): - result = True - elif (w_type == r_type == 'record' and - DatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - result = True - elif (w_type == r_type == 'error' and - DatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - result = True - elif w_type == r_type == 'request': - return True - elif (w_type == r_type == 'fixed' and - DatumReader.check_props(writer_schema, reader_schema, - ['fullname', 'size'])): - result = True - elif (w_type == r_type == 'enum' and - DatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - result = True - elif (w_type == r_type == 'map' and - DatumReader.check_props(writer_schema.values, - reader_schema.values, ['type'])): - result = True - elif (w_type == r_type == 'array' and - DatumReader.check_props(writer_schema.items, - reader_schema.items, ['type'])): - result = True - - # Handle schema promotion - elif w_type == 'int' and r_type in ['long', 'float', 'double']: - result = True - elif w_type == 'long' and r_type in ['float', 'double']: - result = True - elif w_type == 'float' and r_type == 'double': - result = True - else: - result = False - return result - - def __init__(self, writer_schema=None, reader_schema=None): + def __init__(self, writer_schema=None): """ As defined in the Avro specification, we call the schema encoded - in the data the "writer's schema", and the schema expected by the - reader the "reader's schema". + in the data the "writer's schema". """ self._writer_schema = writer_schema - self._reader_schema = reader_schema # read/write properties def set_writer_schema(self, writer_schema): @@ -279,32 +220,10 @@ def set_writer_schema(self, writer_schema): writer_schema = property(lambda self: self._writer_schema, set_writer_schema) - def set_reader_schema(self, reader_schema): - self._reader_schema = reader_schema - - reader_schema = property(lambda self: self._reader_schema, - set_reader_schema) - def read(self, decoder): - if self.reader_schema is None: - self.reader_schema = self.writer_schema - return self.read_data(self.writer_schema, self.reader_schema, decoder) - - def read_data(self, writer_schema, reader_schema, decoder): - # schema matching - if not DatumReader.match_schemas(writer_schema, reader_schema): - fail_msg = 'Schemas do not match.' - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - - # schema resolution: reader's schema is a union, writer's schema is not - if (writer_schema.type not in ['union', 'error_union'] - and reader_schema.type in ['union', 'error_union']): - for s in reader_schema.schemas: - if DatumReader.match_schemas(writer_schema, s): - return self.read_data(writer_schema, s, decoder) - fail_msg = 'Schemas do not match.' - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + return self.read_data(self.writer_schema, decoder) + def read_data(self, writer_schema, decoder): # function dispatch for reading data based on type of writer's schema if writer_schema.type == 'null': result = decoder.read_null() @@ -325,15 +244,15 @@ def read_data(self, writer_schema, reader_schema, decoder): elif writer_schema.type == 'fixed': result = self.read_fixed(writer_schema, decoder) elif writer_schema.type == 'enum': - result = self.read_enum(writer_schema, reader_schema, decoder) + result = self.read_enum(writer_schema, decoder) elif writer_schema.type == 'array': - result = self.read_array(writer_schema, reader_schema, decoder) + result = self.read_array(writer_schema, decoder) elif writer_schema.type == 'map': - result = self.read_map(writer_schema, reader_schema, decoder) + result = self.read_map(writer_schema, decoder) elif writer_schema.type in ['union', 'error_union']: - result = self.read_union(writer_schema, reader_schema, decoder) + result = self.read_union(writer_schema, decoder) elif writer_schema.type in ['record', 'error', 'request']: - result = self.read_record(writer_schema, reader_schema, decoder) + result = self.read_record(writer_schema, decoder) else: fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type raise schema.AvroException(fail_msg) @@ -389,7 +308,7 @@ def skip_fixed(writer_schema, decoder): return decoder.skip(writer_schema.size) @staticmethod - def read_enum(writer_schema, reader_schema, decoder): + def read_enum(writer_schema, decoder): """ An enum is encoded by a int, representing the zero-based position of the symbol in the schema. @@ -399,21 +318,15 @@ def read_enum(writer_schema, reader_schema, decoder): if index_of_symbol >= len(writer_schema.symbols): fail_msg = "Can't access enum index %d for enum with %d symbols" \ % (index_of_symbol, len(writer_schema.symbols)) - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + raise SchemaResolutionException(fail_msg, writer_schema) read_symbol = writer_schema.symbols[index_of_symbol] - - # schema resolution - if read_symbol not in reader_schema.symbols: - fail_msg = "Symbol %s not present in Reader's Schema" % read_symbol - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - return read_symbol @staticmethod def skip_enum(decoder): return decoder.skip_int() - def read_array(self, writer_schema, reader_schema, decoder): + def read_array(self, writer_schema, decoder): """ Arrays are encoded as a series of blocks. @@ -435,8 +348,7 @@ def read_array(self, writer_schema, reader_schema, decoder): block_count = -block_count decoder.read_long() for _ in range(block_count): - read_items.append(self.read_data(writer_schema.items, - reader_schema.items, decoder)) + read_items.append(self.read_data(writer_schema.items, decoder)) block_count = decoder.read_long() return read_items @@ -451,7 +363,7 @@ def skip_array(self, writer_schema, decoder): self.skip_data(writer_schema.items, decoder) block_count = decoder.read_long() - def read_map(self, writer_schema, reader_schema, decoder): + def read_map(self, writer_schema, decoder): """ Maps are encoded as a series of blocks. @@ -474,8 +386,7 @@ def read_map(self, writer_schema, reader_schema, decoder): decoder.read_long() for _ in range(block_count): key = decoder.read_utf8() - read_items[key] = self.read_data(writer_schema.values, - reader_schema.values, decoder) + read_items[key] = self.read_data(writer_schema.values, decoder) block_count = decoder.read_long() return read_items @@ -491,7 +402,7 @@ def skip_map(self, writer_schema, decoder): self.skip_data(writer_schema.values, decoder) block_count = decoder.read_long() - def read_union(self, writer_schema, reader_schema, decoder): + def read_union(self, writer_schema, decoder): """ A union is encoded by first writing a long value indicating the zero-based position within the union of the schema of its value. @@ -502,11 +413,11 @@ def read_union(self, writer_schema, reader_schema, decoder): if index_of_schema >= len(writer_schema.schemas): fail_msg = "Can't access branch index %d for union with %d branches" \ % (index_of_schema, len(writer_schema.schemas)) - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + raise SchemaResolutionException(fail_msg, writer_schema) selected_writer_schema = writer_schema.schemas[index_of_schema] # read data - return self.read_data(selected_writer_schema, reader_schema, decoder) + return self.read_data(selected_writer_schema, decoder) def skip_union(self, writer_schema, decoder): index_of_schema = int(decoder.read_long()) @@ -516,7 +427,7 @@ def skip_union(self, writer_schema, decoder): raise SchemaResolutionException(fail_msg, writer_schema) return self.skip_data(writer_schema.schemas[index_of_schema], decoder) - def read_record(self, writer_schema, reader_schema, decoder): + def read_record(self, writer_schema, decoder): """ A record is encoded by encoding the values of its fields in the order that they are declared. In other words, a record @@ -537,28 +448,10 @@ def read_record(self, writer_schema, reader_schema, decoder): field's value is unset. """ # schema resolution - readers_fields_dict = reader_schema.field_map read_record = {} for field in writer_schema.fields: - readers_field = readers_fields_dict.get(field.name) - if readers_field is not None: - field_val = self.read_data(field.type, readers_field.type, decoder) - read_record[field.name] = field_val - else: - self.skip_data(field.type, decoder) - - # fill in default values - if len(readers_fields_dict) > len(read_record): - writers_fields_dict = writer_schema.field_map - for field_name, field in readers_fields_dict.items(): - if field_name not in writers_fields_dict: - if field.has_default: - field_val = self._read_default_value(field.type, field.default) - read_record[field.name] = field_val - else: - fail_msg = 'No default value for field %s' % field_name - raise SchemaResolutionException(fail_msg, writer_schema, - reader_schema) + field_val = self.read_data(field.type, decoder) + read_record[field.name] = field_val return read_record def skip_record(self, writer_schema, decoder): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py index 07ffe160b823..3f4dc633e9fc 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py @@ -188,68 +188,13 @@ async def skip(self, n): class AsyncDatumReader(object): """Deserialize Avro-encoded data into a Python data structure.""" - @staticmethod - def check_props(schema_one, schema_two, prop_list): - for prop in prop_list: - if getattr(schema_one, prop) != getattr(schema_two, prop): - return False - return True - - @staticmethod - def match_schemas(writer_schema, reader_schema): - w_type = writer_schema.type - r_type = reader_schema.type - if 'union' in [w_type, r_type] or 'error_union' in [w_type, r_type]: - result = True - elif (w_type in schema.PRIMITIVE_TYPES and r_type in schema.PRIMITIVE_TYPES - and w_type == r_type): - result = True - elif (w_type == r_type == 'record' and - AsyncDatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - result = True - elif (w_type == r_type == 'error' and - AsyncDatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - result = True - elif w_type == r_type == 'request': - return True - elif (w_type == r_type == 'fixed' and - AsyncDatumReader.check_props(writer_schema, reader_schema, - ['fullname', 'size'])): - result = True - elif (w_type == r_type == 'enum' and - AsyncDatumReader.check_props(writer_schema, reader_schema, - ['fullname'])): - result = True - elif (w_type == r_type == 'map' and - AsyncDatumReader.check_props(writer_schema.values, - reader_schema.values, ['type'])): - result = True - elif (w_type == r_type == 'array' and - AsyncDatumReader.check_props(writer_schema.items, - reader_schema.items, ['type'])): - result = True - - # Handle schema promotion - elif w_type == 'int' and r_type in ['long', 'float', 'double']: - result = True - elif w_type == 'long' and r_type in ['float', 'double']: - result = True - elif w_type == 'float' and r_type == 'double': - result = True - else: - result = False - return result - - def __init__(self, writer_schema=None, reader_schema=None): + def __init__(self, writer_schema=None): """ As defined in the Avro specification, we call the schema encoded in the data the "writer's schema", and the schema expected by the reader the "reader's schema". """ self._writer_schema = writer_schema - self._reader_schema = reader_schema # read/write properties def set_writer_schema(self, writer_schema): @@ -258,32 +203,10 @@ def set_writer_schema(self, writer_schema): writer_schema = property(lambda self: self._writer_schema, set_writer_schema) - def set_reader_schema(self, reader_schema): - self._reader_schema = reader_schema - - reader_schema = property(lambda self: self._reader_schema, - set_reader_schema) - async def read(self, decoder): - if self.reader_schema is None: - self.reader_schema = self.writer_schema - return await self.read_data(self.writer_schema, self.reader_schema, decoder) - - async def read_data(self, writer_schema, reader_schema, decoder): - # schema matching - if not AsyncDatumReader.match_schemas(writer_schema, reader_schema): - fail_msg = 'Schemas do not match.' - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - - # schema resolution: reader's schema is a union, writer's schema is not - if (writer_schema.type not in ['union', 'error_union'] - and reader_schema.type in ['union', 'error_union']): - for s in reader_schema.schemas: - if AsyncDatumReader.match_schemas(writer_schema, s): - return await self.read_data(writer_schema, s, decoder) - fail_msg = 'Schemas do not match.' - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + return await self.read_data(self.writer_schema, decoder) + async def read_data(self, writer_schema, decoder): # function dispatch for reading data based on type of writer's schema if writer_schema.type == 'null': result = decoder.read_null() @@ -304,15 +227,15 @@ async def read_data(self, writer_schema, reader_schema, decoder): elif writer_schema.type == 'fixed': result = await self.read_fixed(writer_schema, decoder) elif writer_schema.type == 'enum': - result = await self.read_enum(writer_schema, reader_schema, decoder) + result = await self.read_enum(writer_schema, decoder) elif writer_schema.type == 'array': - result = await self.read_array(writer_schema, reader_schema, decoder) + result = await self.read_array(writer_schema, decoder) elif writer_schema.type == 'map': - result = await self.read_map(writer_schema, reader_schema, decoder) + result = await self.read_map(writer_schema, decoder) elif writer_schema.type in ['union', 'error_union']: - result = await self.read_union(writer_schema, reader_schema, decoder) + result = await self.read_union(writer_schema, decoder) elif writer_schema.type in ['record', 'error', 'request']: - result = await self.read_record(writer_schema, reader_schema, decoder) + result = await self.read_record(writer_schema, decoder) else: fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type raise schema.AvroException(fail_msg) @@ -368,7 +291,7 @@ async def skip_fixed(writer_schema, decoder): return await decoder.skip(writer_schema.size) @staticmethod - async def read_enum(writer_schema, reader_schema, decoder): + async def read_enum(writer_schema, decoder): """ An enum is encoded by a int, representing the zero-based position of the symbol in the schema. @@ -378,21 +301,15 @@ async def read_enum(writer_schema, reader_schema, decoder): if index_of_symbol >= len(writer_schema.symbols): fail_msg = "Can't access enum index %d for enum with %d symbols" \ % (index_of_symbol, len(writer_schema.symbols)) - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + raise SchemaResolutionException(fail_msg, writer_schema) read_symbol = writer_schema.symbols[index_of_symbol] - - # schema resolution - if read_symbol not in reader_schema.symbols: - fail_msg = "Symbol %s not present in Reader's Schema" % read_symbol - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) - return read_symbol @staticmethod async def skip_enum(decoder): return await decoder.skip_int() - async def read_array(self, writer_schema, reader_schema, decoder): + async def read_array(self, writer_schema, decoder): """ Arrays are encoded as a series of blocks. @@ -414,8 +331,7 @@ async def read_array(self, writer_schema, reader_schema, decoder): block_count = -block_count await decoder.read_long() for _ in range(block_count): - read_items.append(await self.read_data(writer_schema.items, - reader_schema.items, decoder)) + read_items.append(await self.read_data(writer_schema.items, decoder)) block_count = await decoder.read_long() return read_items @@ -430,7 +346,7 @@ async def skip_array(self, writer_schema, decoder): await self.skip_data(writer_schema.items, decoder) block_count = await decoder.read_long() - async def read_map(self, writer_schema, reader_schema, decoder): + async def read_map(self, writer_schema, decoder): """ Maps are encoded as a series of blocks. @@ -453,8 +369,7 @@ async def read_map(self, writer_schema, reader_schema, decoder): await decoder.read_long() for _ in range(block_count): key = await decoder.read_utf8() - read_items[key] = await self.read_data(writer_schema.values, - reader_schema.values, decoder) + read_items[key] = await self.read_data(writer_schema.values, decoder) block_count = await decoder.read_long() return read_items @@ -470,7 +385,7 @@ async def skip_map(self, writer_schema, decoder): await self.skip_data(writer_schema.values, decoder) block_count = await decoder.read_long() - async def read_union(self, writer_schema, reader_schema, decoder): + async def read_union(self, writer_schema, decoder): """ A union is encoded by first writing a long value indicating the zero-based position within the union of the schema of its value. @@ -481,11 +396,11 @@ async def read_union(self, writer_schema, reader_schema, decoder): if index_of_schema >= len(writer_schema.schemas): fail_msg = "Can't access branch index %d for union with %d branches" \ % (index_of_schema, len(writer_schema.schemas)) - raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) + raise SchemaResolutionException(fail_msg, writer_schema) selected_writer_schema = writer_schema.schemas[index_of_schema] # read data - return await self.read_data(selected_writer_schema, reader_schema, decoder) + return await self.read_data(selected_writer_schema, decoder) async def skip_union(self, writer_schema, decoder): index_of_schema = int(await decoder.read_long()) @@ -495,7 +410,7 @@ async def skip_union(self, writer_schema, decoder): raise SchemaResolutionException(fail_msg, writer_schema) return await self.skip_data(writer_schema.schemas[index_of_schema], decoder) - async def read_record(self, writer_schema, reader_schema, decoder): + async def read_record(self, writer_schema, decoder): """ A record is encoded by encoding the values of its fields in the order that they are declared. In other words, a record @@ -516,28 +431,10 @@ async def read_record(self, writer_schema, reader_schema, decoder): field's value is unset. """ # schema resolution - readers_fields_dict = reader_schema.field_map read_record = {} for field in writer_schema.fields: - readers_field = readers_fields_dict.get(field.name) - if readers_field is not None: - field_val = await self.read_data(field.type, readers_field.type, decoder) - read_record[field.name] = field_val - else: - await self.skip_data(field.type, decoder) - - # fill in default values - if len(readers_fields_dict) > len(read_record): - writers_fields_dict = writer_schema.field_map - for field_name, field in readers_fields_dict.items(): - if field_name not in writers_fields_dict: - if field.has_default: - field_val = self._read_default_value(field.type, field.default) - read_record[field.name] = field_val - else: - fail_msg = 'No default value for field %s' % field_name - raise SchemaResolutionException(fail_msg, writer_schema, - reader_schema) + field_val = await self.read_data(field.type, decoder) + read_record[field.name] = field_val return read_record async def skip_record(self, writer_schema, decoder): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py index d88ad331289e..365f8a236964 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py @@ -172,8 +172,7 @@ def _read_header(self): self.reader.seek(0, 0) # read header into a dict - header = self.datum_reader.read_data( - META_SCHEMA, META_SCHEMA, self.raw_decoder) + header = self.datum_reader.read_data(META_SCHEMA, self.raw_decoder) # check magic number if header.get('magic') != MAGIC: diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py index 2fc33b139533..c8210f8a8b07 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py @@ -133,8 +133,7 @@ async def _read_header(self): await self.reader.seek(0, 0) # read header into a dict - header = await self.datum_reader.read_data( - META_SCHEMA, META_SCHEMA, self.raw_decoder) + header = await self.datum_reader.read_data(META_SCHEMA, self.raw_decoder) # check magic number if header.get('magic') != MAGIC: From c6c0a433a2fc09af2fae5a4317645dfd98541c3f Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Wed, 22 Apr 2020 10:25:08 -0700 Subject: [PATCH 09/13] pr feedback --- .../storage/blob/_shared/avro/avro_io.py | 11 +-- .../blob/_shared/avro/avro_io_async.py | 12 +-- .../storage/blob/_shared/avro/datafile.py | 19 +---- .../blob/_shared/avro/datafile_async.py | 19 +---- .../azure/storage/blob/_shared/avro/schema.py | 78 ++++++++----------- 5 files changed, 45 insertions(+), 94 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py index e7aff7a17424..fa3dc44e20eb 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py @@ -27,7 +27,6 @@ - Schema booleans are implemented as bool. """ -import binascii import json import logging import struct @@ -44,7 +43,6 @@ STRUCT_FLOAT = struct.Struct('I') # big-endian unsigned int # ------------------------------------------------------------------------------ # Exceptions @@ -58,7 +56,7 @@ def __init__(self, fail_msg, writer_schema=None): schema.AvroException.__init__(self, fail_msg) # ------------------------------------------------------------------------------ -# Decoder/Encoder +# Decoder class BinaryDecoder(object): @@ -164,11 +162,6 @@ def read_utf8(self): # PY2 return unicode(input_bytes, "utf-8") # pylint: disable=undefined-variable - def check_crc32(self, input_bytes): - checksum = STRUCT_CRC32.unpack(self.read(4))[0] - if binascii.crc32(input_bytes) & 0xffffffff != checksum: - raise schema.AvroException("Checksum failure") - def skip_null(self): pass @@ -200,7 +193,7 @@ def skip(self, n): # ------------------------------------------------------------------------------ -# DatumReader/Writer +# DatumReader class DatumReader(object): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py index 3f4dc633e9fc..7add3221f104 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py @@ -27,20 +27,19 @@ - Schema booleans are implemented as bool. """ -import binascii import logging import sys from ..avro import schema -from .avro_io import STRUCT_FLOAT, STRUCT_DOUBLE, STRUCT_CRC32, SchemaResolutionException +from .avro_io import STRUCT_FLOAT, STRUCT_DOUBLE, SchemaResolutionException PY3 = sys.version_info[0] == 3 logger = logging.getLogger(__name__) # ------------------------------------------------------------------------------ -# Decoder/Encoder +# Decoder class AsyncBinaryDecoder(object): @@ -146,11 +145,6 @@ async def read_utf8(self): # PY2 return unicode(input_bytes, "utf-8") # pylint: disable=undefined-variable - async def check_crc32(self, input_bytes): - checksum = STRUCT_CRC32.unpack(await self.read(4))[0] - if binascii.crc32(input_bytes) & 0xffffffff != checksum: - raise schema.AvroException("Checksum failure") - def skip_null(self): pass @@ -182,7 +176,7 @@ async def skip(self, n): # ------------------------------------------------------------------------------ -# DatumReader/Writer +# DatumReader class AsyncDatumReader(object): diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py index 365f8a236964..b68da458eae1 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py @@ -74,8 +74,6 @@ class DataFileException(schema.AvroException): class DataFileReader(object): """Read files written by DataFileWriter.""" - # TODO: allow user to specify expected schema? - # TODO: allow user to specify the encoder def __init__(self, reader, datum_reader): """Initializes a new data file reader. @@ -92,7 +90,7 @@ def __init__(self, reader, datum_reader): self._read_header() # ensure codec is valid - avro_codec_raw = self.GetMeta('avro.codec') + avro_codec_raw = self.get_meta('avro.codec') if avro_codec_raw is None: self.codec = "null" else: @@ -103,7 +101,7 @@ def __init__(self, reader, datum_reader): # get ready to read self._block_count = 0 self.datum_reader.writer_schema = ( - schema.parse(self.GetMeta(SCHEMA_KEY).decode('utf-8'))) + schema.parse(self.get_meta(SCHEMA_KEY).decode('utf-8'))) def __enter__(self): return self @@ -146,7 +144,7 @@ def meta(self): def block_count(self): return self._block_count - def GetMeta(self, key): + def get_meta(self, key): """Reports the value of a given metadata key. Args: @@ -156,17 +154,6 @@ def GetMeta(self, key): """ return self._meta.get(key) - def SetMeta(self, key, value): - """Sets a metadata. - - Args: - key: Metadata key (string) to set. - value: Metadata value to set, as bytes. - """ - if isinstance(value, str): - value = value.encode('utf-8') - self._meta[key] = value - def _read_header(self): # seek to the beginning of the file to get magic block self.reader.seek(0, 0) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py index c8210f8a8b07..6076041ffedf 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py @@ -29,8 +29,6 @@ class AsyncDataFileReader(object): """Read files written by DataFileWriter.""" - # TODO: allow user to specify expected schema? - # TODO: allow user to specify the encoder def __init__(self, reader, datum_reader): """Initializes a new data file reader. @@ -52,7 +50,7 @@ async def init(self): await self._read_header() # ensure codec is valid - avro_codec_raw = self.GetMeta('avro.codec') + avro_codec_raw = self.get_meta('avro.codec') if avro_codec_raw is None: self.codec = "null" else: @@ -63,7 +61,7 @@ async def init(self): # get ready to read self._block_count = 0 self.datum_reader.writer_schema = ( - schema.parse(self.GetMeta(SCHEMA_KEY).decode('utf-8'))) + schema.parse(self.get_meta(SCHEMA_KEY).decode('utf-8'))) return self async def __aenter__(self): @@ -107,7 +105,7 @@ def meta(self): def block_count(self): return self._block_count - def GetMeta(self, key): + def get_meta(self, key): """Reports the value of a given metadata key. Args: @@ -117,17 +115,6 @@ def GetMeta(self, key): """ return self._meta.get(key) - def SetMeta(self, key, value): - """Sets a metadata. - - Args: - key: Metadata key (string) to set. - value: Metadata value to set, as bytes. - """ - if isinstance(value, str): - value = value.encode('utf-8') - self._meta[key] = value - async def _read_header(self): # seek to the beginning of the file to get magic block await self.reader.seek(0, 0) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py index a18f79bce677..6832ab4fd2d5 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/schema.py @@ -192,7 +192,7 @@ def props(self): @property def other_props(self): """Returns: the dictionary of non-reserved properties.""" - return dict(FilterKeysOut(items=self._props, keys=SCHEMA_RESERVED_PROPS)) + return dict(filter_keys_out(items=self._props, keys=SCHEMA_RESERVED_PROPS)) def __str__(self): """Returns: the JSON representation of this schema.""" @@ -311,7 +311,7 @@ def default_namespace(self): """Returns: the default namespace, if any, or None.""" return self._default_namespace - def NewWithDefaultNamespace(self, namespace): + def new_with_default_namespace(self, namespace): """Creates a new name tracker from this tracker, but with a new default ns. Args: @@ -321,7 +321,7 @@ def NewWithDefaultNamespace(self, namespace): """ return Names(names=self._names, default_namespace=namespace) - def GetName(self, name, namespace=None): + def get_name(self, name, namespace=None): """Resolves the Avro name according to this name tracker's state. Args: @@ -334,15 +334,7 @@ def GetName(self, name, namespace=None): namespace = self._default_namespace return Name(name=name, namespace=namespace) - def has_name(self, name, namespace=None): - avro_name = self.GetName(name=name, namespace=namespace) - return avro_name.fullname in self._names - - def get_name(self, name, namespace=None): - avro_name = self.GetName(name=name, namespace=namespace) - return self._names.get(avro_name.fullname, None) - - def GetSchema(self, name, namespace=None): + def get_schema(self, name, namespace=None): """Resolves an Avro schema by name. Args: @@ -351,7 +343,7 @@ def GetSchema(self, name, namespace=None): Returns: The schema with the specified name, if any, or None. """ - avro_name = self.GetName(name=name, namespace=namespace) + avro_name = self.get_name(name=name, namespace=namespace) return self._names.get(avro_name.fullname, None) def prune_namespace(self, properties): @@ -372,7 +364,7 @@ def prune_namespace(self, properties): del prunable['namespace'] return prunable - def Register(self, schema): + def register(self, schema): """Registers a new named schema in this tracker. Args: @@ -416,11 +408,11 @@ def __init__( other_props: Optional map of additional properties of the schema. """ assert (data_type in NAMED_TYPES), ('Invalid named type: %r' % data_type) - self._avro_name = names.GetName(name=name, namespace=namespace) + self._avro_name = names.get_name(name=name, namespace=namespace) super(NamedSchema, self).__init__(data_type, other_props) - names.Register(self) + names.register(self) self._props['name'] = self.name if self.namespace: @@ -513,7 +505,6 @@ def __init__( self._type = self._props['type'] = data_type self._name = self._props['name'] = name - # TODO: check to ensure default is valid if has_default: self._props['default'] = default @@ -560,7 +551,7 @@ def props(self): @property def other_props(self): - return FilterKeysOut(items=self._props, keys=FIELD_RESERVED_PROPS) + return filter_keys_out(items=self._props, keys=FIELD_RESERVED_PROPS) def __str__(self): return json.dumps(self.to_json()) @@ -866,7 +857,6 @@ def __init__(self, schemas): Args: schema: collection of error schema. """ - # TODO: check that string isn't already listed explicitly as an error. # Prepend "string" to handle system errors schemas = [PrimitiveSchema(data_type=STRING)] + list(schemas) super(ErrorUnionSchema, self).__init__(schemas=schemas) @@ -890,7 +880,7 @@ class RecordSchema(NamedSchema): """Schema of a record.""" @staticmethod - def _MakeField(index, field_desc, names): + def _make_field(index, field_desc, names): """Builds field schemas from a list of field JSON descriptors. Args: @@ -899,12 +889,12 @@ def _MakeField(index, field_desc, names): Return: The field schema. """ - field_schema = SchemaFromJSONData( + field_schema = schema_from_json_data( json_data=field_desc['type'], names=names, ) other_props = ( - dict(FilterKeysOut(items=field_desc, keys=FIELD_RESERVED_PROPS))) + dict(filter_keys_out(items=field_desc, keys=FIELD_RESERVED_PROPS))) return Field( data_type=field_schema, name=field_desc['name'], @@ -917,7 +907,7 @@ def _MakeField(index, field_desc, names): ) @staticmethod - def MakeFieldList(field_desc_list, names): + def make_field_list(field_desc_list, names): """Builds field schemas from a list of field JSON descriptors. Guarantees field name unicity. @@ -929,10 +919,10 @@ def MakeFieldList(field_desc_list, names): Field schemas. """ for index, field_desc in enumerate(field_desc_list): - yield RecordSchema._MakeField(index, field_desc, names) + yield RecordSchema._make_field(index, field_desc, names) @staticmethod - def _MakeFieldMap(fields): + def _make_field_map(fields): """Builds the field map. Guarantees field name unicity. @@ -997,8 +987,8 @@ def __init__( 'Invalid record type: %r.' % record_type) if record_type in [RECORD, ERROR]: - avro_name = names.GetName(name=name, namespace=namespace) - nested_names = names.NewWithDefaultNamespace(namespace=avro_name.namespace) + avro_name = names.get_name(name=name, namespace=namespace) + nested_names = names.new_with_default_namespace(namespace=avro_name.namespace) elif record_type == REQUEST: # Protocol request has no name: no need to change default namespace: nested_names = names @@ -1009,7 +999,7 @@ def __init__( assert make_fields is None self._fields = tuple(fields) - self._field_map = RecordSchema._MakeFieldMap(self._fields) + self._field_map = RecordSchema._make_field_map(self._fields) self._props['fields'] = fields if doc is not None: @@ -1049,7 +1039,7 @@ def __eq__(self, that): # Module functions -def FilterKeysOut(items, keys): +def filter_keys_out(items, keys): """Filters a collection of (key, value) items. Exclude any item whose key belongs to keys. @@ -1069,12 +1059,12 @@ def FilterKeysOut(items, keys): # ------------------------------------------------------------------------------ -def _SchemaFromJSONString(json_string, names): +def _schema_from_json_string(json_string, names): if json_string in PRIMITIVE_TYPES: return PrimitiveSchema(data_type=json_string) # Look for a known named schema: - schema = names.GetSchema(name=json_string) + schema = names.get_schema(name=json_string) if schema is None: raise SchemaParseException( 'Unknown named schema %r, known names: %r.' @@ -1082,21 +1072,21 @@ def _SchemaFromJSONString(json_string, names): return schema -def _SchemaFromJSONArray(json_array, names): +def _schema_from_json_array(json_array, names): def MakeSchema(desc): - return SchemaFromJSONData(json_data=desc, names=names) + return schema_from_json_data(json_data=desc, names=names) return UnionSchema(map(MakeSchema, json_array)) -def _SchemaFromJSONObject(json_object, names): +def _schema_from_json_object(json_object, names): data_type = json_object.get('type') if data_type is None: raise SchemaParseException( 'Avro schema JSON descriptor has no "type" property: %r' % json_object) other_props = dict( - FilterKeysOut(items=json_object, keys=SCHEMA_RESERVED_PROPS)) + filter_keys_out(items=json_object, keys=SCHEMA_RESERVED_PROPS)) if data_type in PRIMITIVE_TYPES: # FIXME should not ignore other properties @@ -1117,7 +1107,7 @@ def _SchemaFromJSONObject(json_object, names): field_desc_list = json_object.get('fields', ()) def MakeFields(names): - return tuple(RecordSchema.MakeFieldList(field_desc_list, names)) + return tuple(RecordSchema.make_field_list(field_desc_list, names)) result = RecordSchema( name=name, @@ -1141,7 +1131,7 @@ def MakeFields(names): 'Invalid array schema descriptor with no "items" : %r.' % json_object) result = ArraySchema( - items=SchemaFromJSONData(items_desc, names), + items=schema_from_json_data(items_desc, names), other_props=other_props, ) @@ -1152,7 +1142,7 @@ def MakeFields(names): 'Invalid map schema descriptor with no "values" : %r.' % json_object) result = MapSchema( - values=SchemaFromJSONData(values_desc, names=names), + values=schema_from_json_data(values_desc, names=names), other_props=other_props, ) @@ -1160,7 +1150,7 @@ def MakeFields(names): error_desc_list = json_object.get('declared_errors') assert error_desc_list is not None error_schemas = map( - lambda desc: SchemaFromJSONData(desc, names=names), + lambda desc: schema_from_json_data(desc, names=names), error_desc_list) result = ErrorUnionSchema(schemas=error_schemas) @@ -1174,13 +1164,13 @@ def MakeFields(names): # Parsers for the JSON data types: _JSONDataParserTypeMap = { - _str: _SchemaFromJSONString, - list: _SchemaFromJSONArray, - dict: _SchemaFromJSONObject, + _str: _schema_from_json_string, + list: _schema_from_json_array, + dict: _schema_from_json_object, } -def SchemaFromJSONData(json_data, names=None): +def schema_from_json_data(json_data, names=None): """Builds an Avro Schema from its JSON descriptor. Args: @@ -1228,4 +1218,4 @@ def parse(json_string): names = Names() # construct the Avro Schema object - return SchemaFromJSONData(json_data, names) + return schema_from_json_data(json_data, names) From 319f52f0e8a780af2627184936658c36fb1eb22d Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Wed, 22 Apr 2020 10:32:53 -0700 Subject: [PATCH 10/13] trim unused code. --- .../storage/blob/_shared/avro/avro_io.py | 44 ------------------- .../blob/_shared/avro/avro_io_async.py | 44 ------------------- 2 files changed, 88 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py index fa3dc44e20eb..d830da11fd93 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py @@ -451,50 +451,6 @@ def skip_record(self, writer_schema, decoder): for field in writer_schema.fields: self.skip_data(field.type, decoder) - def _read_default_value(self, field_schema, default_value): - """ - Basically a JSON Decoder? - """ - if field_schema.type == 'null': - result = None - elif field_schema.type == 'boolean': - result = bool(default_value) - elif field_schema.type == 'int': - result = int(default_value) - elif field_schema.type == 'long': - result = int(default_value) - elif field_schema.type in ['float', 'double']: - result = float(default_value) - elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']: - result = default_value - elif field_schema.type == 'array': - read_array = [] - for json_val in default_value: - item_val = self._read_default_value(field_schema.items, json_val) - read_array.append(item_val) - result = read_array - elif field_schema.type == 'map': - read_map = {} - for key, json_val in default_value.items(): - map_val = self._read_default_value(field_schema.values, json_val) - read_map[key] = map_val - result = read_map - elif field_schema.type in ['union', 'error_union']: - result = self._read_default_value(field_schema.schemas[0], default_value) - elif field_schema.type == 'record': - read_record = {} - for field in field_schema.fields: - json_val = default_value.get(field.name) - if json_val is None: - json_val = field.default - field_val = self._read_default_value(field.type, json_val) - read_record[field.name] = field_val - result = read_record - else: - fail_msg = 'Unknown type: %s' % field_schema.type - raise schema.AvroException(fail_msg) - return result - # ------------------------------------------------------------------------------ diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py index 7add3221f104..cd466a46e9cc 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py @@ -435,50 +435,6 @@ async def skip_record(self, writer_schema, decoder): for field in writer_schema.fields: await self.skip_data(field.type, decoder) - def _read_default_value(self, field_schema, default_value): - """ - Basically a JSON Decoder? - """ - if field_schema.type == 'null': - result = None - elif field_schema.type == 'boolean': - result = bool(default_value) - elif field_schema.type == 'int': - result = int(default_value) - elif field_schema.type == 'long': - result = int(default_value) - elif field_schema.type in ['float', 'double']: - result = float(default_value) - elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']: - result = default_value - elif field_schema.type == 'array': - read_array = [] - for json_val in default_value: - item_val = self._read_default_value(field_schema.items, json_val) - read_array.append(item_val) - result = read_array - elif field_schema.type == 'map': - read_map = {} - for key, json_val in default_value.items(): - map_val = self._read_default_value(field_schema.values, json_val) - read_map[key] = map_val - result = read_map - elif field_schema.type in ['union', 'error_union']: - result = self._read_default_value(field_schema.schemas[0], default_value) - elif field_schema.type == 'record': - read_record = {} - for field in field_schema.fields: - json_val = default_value.get(field.name) - if json_val is None: - json_val = field.default - field_val = self._read_default_value(field.type, json_val) - read_record[field.name] = field_val - result = read_record - else: - fail_msg = 'Unknown type: %s' % field_schema.type - raise schema.AvroException(fail_msg) - return result - # ------------------------------------------------------------------------------ From be783ea3ec1344864f8bd13c5a56cd506ebce8d1 Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Wed, 22 Apr 2020 12:42:03 -0700 Subject: [PATCH 11/13] pr feedback. --- .../azure/storage/blob/_shared/avro/avro_io.py | 8 +++++++- .../azure/storage/blob/_shared/avro/avro_io_async.py | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py index d830da11fd93..93a5c134849a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io.py @@ -100,7 +100,13 @@ def read_boolean(self): a boolean is written as a single byte whose value is either 0 (false) or 1 (true). """ - return ord(self.read(1)) == 1 + b = ord(self.read(1)) + if b == 1: + return True + if b == 0: + return False + fail_msg = "Invalid value for boolean: %s" % b + raise schema.AvroException(fail_msg) def read_int(self): """ diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py index cd466a46e9cc..e9812163795f 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/avro_io_async.py @@ -83,7 +83,13 @@ async def read_boolean(self): a boolean is written as a single byte whose value is either 0 (false) or 1 (true). """ - return ord(await self.read(1)) == 1 + b = ord(await self.read(1)) + if b == 1: + return True + if b == 0: + return False + fail_msg = "Invalid value for boolean: %s" % b + raise schema.AvroException(fail_msg) async def read_int(self): """ From 3f6b1436722bd9944d5adb52a6fab5cf535738a4 Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Wed, 22 Apr 2020 13:02:46 -0700 Subject: [PATCH 12/13] simplify skip sync in next. --- .../azure/storage/blob/_shared/avro/datafile.py | 8 ++------ .../azure/storage/blob/_shared/avro/datafile_async.py | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py index b68da458eae1..85b7dadb9854 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile.py @@ -200,16 +200,12 @@ def _skip_sync(self): raise StopIteration if proposed_sync_marker != self.sync_marker: self.reader.seek(-SYNC_SIZE, 1) - return False - return True def __next__(self): """Return the next datum in the file.""" if self.block_count == 0: - if self._skip_sync(): - self._read_block_header() - else: - self._read_block_header() + self._skip_sync() + self._read_block_header() datum = self.datum_reader.read(self.datum_decoder) self._block_count -= 1 diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py index 6076041ffedf..75f0539aa1f7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/avro/datafile_async.py @@ -153,16 +153,12 @@ async def _skip_sync(self): raise StopAsyncIteration if proposed_sync_marker != self.sync_marker: await self.reader.seek(-SYNC_SIZE, 1) - return False - return True async def __anext__(self): """Return the next datum in the file.""" if self.block_count == 0: - if await self._skip_sync(): - await self._read_block_header() - else: - await self._read_block_header() + await self._skip_sync() + await self._read_block_header() datum = await self.datum_reader.read(self.datum_decoder) self._block_count -= 1 From 4ce5a8b7fcf1e05748340d04b75dec45f8e572ed Mon Sep 17 00:00:00 2001 From: Kamil Sobol <61715331+kasobol-msft@users.noreply.github.com> Date: Thu, 23 Apr 2020 08:46:59 -0700 Subject: [PATCH 13/13] move avro tests from _shared. --- .../tests/{_shared => }/avro/__init__.py | 0 .../{_shared => }/avro/samples/changeFeed.avro | Bin .../{_shared => }/avro/samples/test_deflate_0.avro | Bin .../{_shared => }/avro/samples/test_deflate_1.avro | Bin .../{_shared => }/avro/samples/test_deflate_10.avro | Bin .../{_shared => }/avro/samples/test_deflate_11.avro | Bin .../{_shared => }/avro/samples/test_deflate_12.avro | Bin .../{_shared => }/avro/samples/test_deflate_13.avro | Bin .../{_shared => }/avro/samples/test_deflate_14.avro | Bin .../{_shared => }/avro/samples/test_deflate_2.avro | Bin .../{_shared => }/avro/samples/test_deflate_3.avro | Bin .../{_shared => }/avro/samples/test_deflate_4.avro | Bin .../{_shared => }/avro/samples/test_deflate_5.avro | Bin .../{_shared => }/avro/samples/test_deflate_6.avro | Bin .../{_shared => }/avro/samples/test_deflate_7.avro | Bin .../{_shared => }/avro/samples/test_deflate_8.avro | Bin .../{_shared => }/avro/samples/test_deflate_9.avro | Bin .../{_shared => }/avro/samples/test_null_0.avro | Bin .../{_shared => }/avro/samples/test_null_1.avro | Bin .../{_shared => }/avro/samples/test_null_10.avro | Bin .../{_shared => }/avro/samples/test_null_11.avro | Bin .../{_shared => }/avro/samples/test_null_12.avro | Bin .../{_shared => }/avro/samples/test_null_13.avro | Bin .../{_shared => }/avro/samples/test_null_14.avro | Bin .../{_shared => }/avro/samples/test_null_2.avro | Bin .../{_shared => }/avro/samples/test_null_3.avro | Bin .../{_shared => }/avro/samples/test_null_4.avro | Bin .../{_shared => }/avro/samples/test_null_5.avro | Bin .../{_shared => }/avro/samples/test_null_6.avro | Bin .../{_shared => }/avro/samples/test_null_7.avro | Bin .../{_shared => }/avro/samples/test_null_8.avro | Bin .../{_shared => }/avro/samples/test_null_9.avro | Bin .../tests/{_shared => }/avro/test_avro.py | 0 .../tests/{_shared => }/avro/test_avro_async.py | 0 34 files changed, 0 insertions(+), 0 deletions(-) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/__init__.py (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/changeFeed.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_0.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_1.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_10.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_11.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_12.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_13.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_14.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_2.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_3.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_4.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_5.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_6.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_7.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_8.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_deflate_9.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_0.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_1.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_10.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_11.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_12.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_13.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_14.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_2.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_3.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_4.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_5.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_6.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_7.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_8.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/samples/test_null_9.avro (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/test_avro.py (100%) rename sdk/storage/azure-storage-blob/tests/{_shared => }/avro/test_avro_async.py (100%) diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/__init__.py b/sdk/storage/azure-storage-blob/tests/avro/__init__.py similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/__init__.py rename to sdk/storage/azure-storage-blob/tests/avro/__init__.py diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/changeFeed.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/changeFeed.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/changeFeed.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/changeFeed.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_0.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_0.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_0.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_0.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_1.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_1.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_1.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_1.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_10.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_10.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_10.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_10.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_11.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_11.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_11.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_11.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_12.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_12.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_12.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_12.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_13.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_13.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_13.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_13.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_14.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_14.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_14.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_14.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_2.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_2.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_2.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_2.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_3.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_3.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_3.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_3.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_4.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_4.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_4.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_4.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_5.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_5.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_5.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_5.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_6.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_6.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_6.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_6.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_7.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_7.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_7.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_7.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_8.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_8.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_8.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_8.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_9.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_9.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_deflate_9.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_deflate_9.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_0.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_0.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_0.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_0.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_1.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_1.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_1.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_1.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_10.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_10.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_10.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_10.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_11.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_11.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_11.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_11.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_12.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_12.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_12.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_12.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_13.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_13.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_13.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_13.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_14.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_14.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_14.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_14.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_2.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_2.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_2.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_2.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_3.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_3.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_3.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_3.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_4.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_4.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_4.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_4.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_5.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_5.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_5.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_5.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_6.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_6.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_6.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_6.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_7.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_7.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_7.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_7.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_8.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_8.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_8.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_8.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_9.avro b/sdk/storage/azure-storage-blob/tests/avro/samples/test_null_9.avro similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/samples/test_null_9.avro rename to sdk/storage/azure-storage-blob/tests/avro/samples/test_null_9.avro diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py b/sdk/storage/azure-storage-blob/tests/avro/test_avro.py similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro.py rename to sdk/storage/azure-storage-blob/tests/avro/test_avro.py diff --git a/sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py b/sdk/storage/azure-storage-blob/tests/avro/test_avro_async.py similarity index 100% rename from sdk/storage/azure-storage-blob/tests/_shared/avro/test_avro_async.py rename to sdk/storage/azure-storage-blob/tests/avro/test_avro_async.py