From 6c848a2701c6917cecf52723f7b3348539d06a26 Mon Sep 17 00:00:00 2001 From: "Tsung-Sheng Huang (Hi-Tech Talents LLC)" Date: Sun, 23 Jun 2019 12:06:19 -0700 Subject: [PATCH] Updated CharTokenizer.py example --- src/python/nimbusml/examples/CharTokenizer.py | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/src/python/nimbusml/examples/CharTokenizer.py b/src/python/nimbusml/examples/CharTokenizer.py index 8762ce6a..56c6ab04 100644 --- a/src/python/nimbusml/examples/CharTokenizer.py +++ b/src/python/nimbusml/examples/CharTokenizer.py @@ -1,25 +1,45 @@ ############################################################################### # CharTokenizer import numpy -from nimbusml import FileDataStream, DataSchema +from nimbusml import FileDataStream, DataSchema, Pipeline from nimbusml.datasets import get_dataset +from nimbusml.preprocessing import FromKey +from nimbusml.preprocessing.text import CharTokenizer +from nimbusml.preprocessing.schema import ColumnSelector +from nimbusml.feature_extraction.text import WordEmbedding # data input (as a FileDataStream) -path = get_dataset('infert').as_filepath() +path = get_dataset('wiki_detox_train').as_filepath() file_schema = DataSchema.read_schema( - path, sep=',', numeric_dtype=numpy.float32) + path, sep='\t', numeric_dtype=numpy.float32) data = FileDataStream(path, schema=file_schema) -print(data.schema) - -# Section below throws "System.Runtime.InteropServices.SEHException" -# Logged in https://github.com/Microsoft/NimbusML/issues/31 - -# # transform usage -# xf = CharTokenizer(columns={'id_1': 'id', 'edu_1': 'education'}) -# -# # fit and transform -# features = xf.fit_transform(data) -# -# # print features -# print(features.head()) +print(data.head()) + +# Sentiment SentimentText +# 0 1.0 ==RUDE== Dude, you are rude upload that carl p... +# 1 1.0 == OK! == IM GOING TO VANDALIZE WILD ONES WIK... +# 2 1.0 Stop trolling, zapatancas, calling me a liar m... +# 3 1.0 ==You're cool== You seem like a really cool g... +# 4 1.0 ::::: Why are you threatening me? I'm not bein... + +# After using Character Tokenizer, it will convert the vector of Char to Key type. +# Use FromKey to retrieve the data from Key first, then send into WordEmbedding. + +pipe = Pipeline([ + CharTokenizer(columns={'SentimentText_Transform': 'SentimentText'}), + FromKey(columns={'SentimentText_FromKey': 'SentimentText_Transform'}), + WordEmbedding(model_kind='GloVe50D', columns={'Feature': 'SentimentText_FromKey'}), + ColumnSelector(columns=['Sentiment', 'SentimentText', 'Feature']) + ]) + +print(pipe.fit_transform(data).head()) + +# Sentiment ... Feature.149 +# 0 1.0 ... 2.67440 +# 1 1.0 ... 0.78858 +# 2 1.0 ... 2.67440 +# 3 1.0 ... 2.67440 +# 4 1.0 ... 2.67440 + +# [5 rows x 152 columns]