ML Services のサンプル追加

ML Services/01.sp_execute_external_script/01.ML Service Python.sql

@@ -0,0 +1,20 @@
+exec sp_execute_external_script 
+@language =N'Python', 
+@script=N'OutputDataSet = InputDataSet', 
+@input_data_1 = N'SELECT name,type_desc FROM sys.objects'
+WITH RESULT SETS ((
+name sysname,
+type_desc nvarchar(60)
+))
+
+exec sp_execute_external_script  
+@language =N'Python',  
+@script=N'
+import os
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+					
+import tensorflow as tf
+hello = tf.constant("Hello, TensorFlow!")
+sess = tf.Session()
+print(sess.run(hello))
+'

ML Services/02.PREDICT 関数/01.モデルのコピー.sql

@@ -0,0 +1,12 @@
+USE [TutorialDB]
+GO
+
+SELECT * FROM [WIN01.Techsummit.local].[TutorialDB].[dbo].[rental_models]
+
+DELETE FROM rental_models WHERE model_name = 'linear_model'
+GO
+INSERT INTO [rental_models]
+SELECT * FROM [WIN01.Techsummit.local].[TutorialDB].[dbo].[rental_models]
+GO
+
+SELECT * FROM [rental_models]

ML Services/02.PREDICT 関数/02.クエリの実行.sql

@@ -0,0 +1,19 @@
+USE MLDB
+GO
+
+SELECT * FROM rental_models
+GO
+
+-- テーブルから学習済みモデルを取得
+DECLARE @model varbinary(max) = 
+(SELECT native_model FROM rental_models WHERE model_name = 'linear_model');
+
+-- PREDICT 関数を使用した学習済みモデルの利用
+WITH d AS (SELECT TOP 100 * FROM dbo.rental_data WHERE Year = 2015 ORDER BY Year, Month, Day)
+SELECT d.Year,  d.Month, d.Day, d.WeekDay, d.Holiday, d.snow, d.RentalCount, 
+CAST(p.RentalCount_Pred AS int) as RentalCount_Pred
+FROM 
+PREDICT(MODEL = @model, DATA =d AS d) 
+WITH(RentalCount_Pred float) AS p
+ORDER BY Year, Month, day
+GO

ML Services/03.VS Code の利用/00.NativeScoring_事前準備.sql

@@ -0,0 +1,24 @@
+USE TutorialDB;
+GO
+DROP TABLE IF EXISTS rental_models;
+GO
+CREATE TABLE rental_models (
+                model_name VARCHAR(30) NOT NULL DEFAULT('default model'),
+                lang VARCHAR(30),
+				model VARBINARY(MAX),
+				native_model VARBINARY(MAX),
+				PRIMARY KEY (model_name, lang)
+);
+GO
+
+DECLARE @model varbinary(max) = 
+(SELECT * FROM 
+OPENROWSET(BULK N'c:\temp\trained_model.pickle', SINGLE_BLOB) AS t);
+
+WITH d AS (SELECT * FROM dbo.rental_data WHERE Year = 2013)
+
+SELECT d.*, p.* 
+FROM 
+PREDICT(MODEL = @model, DATA =d AS d) 
+WITH(RentalCount_Pred float) AS p;
+GO

ML Services/03.VS Code の利用/01.NativeScoreing_Linux.py

@@ -0,0 +1,36 @@
+from revoscalepy import rx_lin_mod, rx_serialize_model, rx_summary
+import pandas as pd
+import pyodbc
+import os
+
+conn_str = 'Driver=SQL Server;Server=<Server Name>;Database=MLDB;Uid=<User Name>;Pwd=<Password>;'
+cnxn = pyodbc.connect(conn_str)
+cnxn.setencoding("utf-8")
+inputsql = 'select "RentalCount", "Year", "Month", "Day", "WeekDay", "Snow", "Holiday", "FWeekDay" from dbo.rental_data where Year < 2015'
+rental_train_data = pd.read_sql(inputsql, cnxn)
+
+rental_train_data["Holiday"] = rental_train_data["Holiday"].astype("category")
+rental_train_data["Snow"] = rental_train_data["Snow"].astype("category")
+rental_train_data["WeekDay"] = rental_train_data["WeekDay"].astype("category")
+
+linmod_model = rx_lin_mod("RentalCount ~ Month + Day + WeekDay + Snow + Holiday", data = rental_train_data)
+trained_model = rx_serialize_model(linmod_model, realtime_scoring_only = True)
+
+print(rx_summary("RentalCount ~ Month + Day + WeekDay + Snow + Holiday", rental_train_data))
+
+# Dump learned model to file
+with open(r'c:\model\trained_model.pickle', mode='wb') as f:
+    f.write(trained_model)
+
+# Dump learned model to Table
+cursor=cnxn.cursor()
+cursor.execute(\
+'''
+MERGE rental_models AS target
+USING (SELECT ? as model_name) AS source
+ON(target.model_name = source.model_name)
+WHEN MATCHED THEN UPDATE SET native_model = ?
+WHEN NOT MATCHED BY TARGET THEN INSERT (model_name, lang, native_model) VALUES(?,?,?);
+''', \
+("linear_model", trained_model, "linear_model", "Python", trained_model))
+cnxn.commit()

ML Services/03.VS Code の利用/02.NativeScoring.sql

@@ -0,0 +1,23 @@
+
+-- テーブルのデータ
+SELECT TOP 100 * FROM dbo.rental_data WHERE Year = 2015 ORDER BY Year, Month, Day
+GO
+
+-- 学習済みモデルの表示
+SELECT * FROM rental_models WHERE model_name = 'linear_model'
+GO
+
+-- PREDICT 関数を使用した学習済みモデルの利用
+DECLARE @model varbinary(max) = 
+(SELECT native_model FROM rental_models WHERE model_name = 'linear_model');
+
+WITH d AS (SELECT TOP 100 *
+FROM dbo.rental_data WHERE Year = 2015 ORDER BY Year, Month, Day)
+SELECT d.Year,  d.Month, d.Day, d.RentalCount, 
+CAST(p.RentalCount_Pred AS int) as RentalCount_Pred,
+d.WeekDay, d.Holiday, d.snow
+FROM 
+PREDICT(MODEL = @model, DATA =d AS d) 
+WITH(RentalCount_Pred float) AS p
+ORDER BY Year, Month, day
+GO

ML Services/03.VS Code の利用/NativeScoreing_LocalContext.py

@@ -0,0 +1,44 @@
+'''
+# https://microsoft.github.io/sql-ml-tutorials/python/rentalprediction/
+# https://docs.microsoft.com/en-us/machine-learning-server/install/python-libraries-interpreter
+[Environment]::SetEnvironmentVariable('UID', '<SQL Login>', 'User')
+[Environment]::SetEnvironmentVariable('SQLPass', '<SQL Login Password>', 'User')
+'''
+import os
+
+from revoscalepy import rx_lin_mod, rx_serialize_model, rx_dtree, rx_summary
+from pandas import Categorical
+import pandas as pd
+import pyodbc
+
+model_type = "linear"
+conn_str = 'Driver=SQL Server;Server=<Server Name>;Database=MLDB;Uid=<User Name>;Pwd=<Password>;'
+cnxn = pyodbc.connect(conn_str)
+inputsql = 'select "RentalCount", "Year", "Month", "Day", "WeekDay", "Snow", "Holiday", "FWeekDay" from dbo.rental_data where Year < 2015'
+rental_train_data = pd.read_sql(inputsql, cnxn)
+
+# Used to sample data
+# train = rental_train_data.sample(frac=0.8, random_state=1)
+# test = rental_train_data.loc[~ rental_train_data.index.isin(train.index)]
+# print("Train {0} / test {1}".format(len(train), len(test)))
+
+rental_train_data["Holiday"] = rental_train_data["Holiday"].astype("category")
+rental_train_data["Snow"] = rental_train_data["Snow"].astype("category")
+rental_train_data["WeekDay"] = rental_train_data["WeekDay"].astype("category")
+
+if model_type == "linear":
+    linmod_model = rx_lin_mod("RentalCount ~ Month + Day + WeekDay + Snow + Holiday", data = rental_train_data)
+    trained_model = rx_serialize_model(linmod_model, realtime_scoring_only = True)
+if model_type == "dtree":
+	dtree_model = rx_dtree("RentalCount ~ Month + Day + WeekDay + Snow + Holiday", data = rental_train_data)
+	trained_model = rx_serialize_model(dtree_model, realtime_scoring_only = True)
+
+print(rx_summary("~ Month + Day + WeekDay + Snow + Holiday", rental_train_data))
+
+# Dump learned model to file
+with open(r'c:\temp\trained_model.pickle', mode='wb') as f:
+    f.write(trained_model)
+
+cursor=cnxn.cursor()
+cursor.execute("INSERT INTO rental_models(model_name, lang, native_model) VALUES(?, ?, ?)", (model_type + "_model", "Python", trained_model))
+cnxn.commit()

ML Services/03.VS Code の利用/NativeScoring_RemoteContext.py

@@ -0,0 +1,63 @@
+'''
+# https://microsoft.github.io/sql-ml-tutorials/python/rentalprediction/
+# https://docs.microsoft.com/en-us/machine-learning-server/install/python-libraries-interpreter
+[Environment]::SetEnvironmentVariable('UID', '<SQL Login>', 'User')
+[Environment]::SetEnvironmentVariable('SQLPass', '<SQL Login Password>', 'User')
+'''
+import os
+
+from revoscalepy import rx_lin_mod, rx_serialize_model, rx_dtree, rx_summary
+
+from revoscalepy import RxComputeContext, RxInSqlServer, RxSqlServerData
+from revoscalepy import RxLocalSeq, rx_get_compute_context, rx_set_compute_context
+from revoscalepy import rx_import
+from pandas import Categorical
+import pandas as pd
+import pyodbc
+
+model_type = "linear"
+conn_str = 'Driver=SQL Server;Server=<Server Name>;Database=MLDB;Uid=<User Name>;Pwd=<Password>;'
+inputsql = 'select "RentalCount", "Year", "Month", "Day", "WeekDay", "Snow", "Holiday", "FWeekDay" from dbo.rental_data where Year < 2015'
+
+cc = RxInSqlServer(
+     connection_string = conn_str,
+     num_tasks = 1,
+     auto_cleanup = False
+)
+local_cc = RxLocalSeq()
+previous_cc = rx_set_compute_context(cc)
+rx_get_compute_context()
+
+column_info = {
+         "Year" : { "type" : "integer" },
+         "Month" : { "type" : "integer" },
+         "Day" : { "type" : "integer" },
+         "RentalCount" : { "type" : "integer" },
+         "WeekDay" : {
+             "type" : "factor",
+             "levels" : ["1", "2", "3", "4", "5", "6", "7"]
+         },
+         "Holiday" : {
+             "type" : "factor",
+             "levels" : ["1", "0"]
+         },
+         "Snow" : {
+             "type" : "factor",
+             "levels" : ["1", "0"]
+         }
+     }
+data_source = RxSqlServerData(sql_query=inputsql, connection_string=conn_str, column_info=column_info)
+
+linmod_model = rx_lin_mod("RentalCount ~ Month + Day + WeekDay + Snow + Holiday", data = data_source, computeContext = cc)
+trained_model = rx_serialize_model(linmod_model, realtime_scoring_only = True)
+
+with open(r'c:\temp\trained_model.pickle', mode='wb') as f:
+    f.write(trained_model)
+
+print(rx_summary("RentalCount ~ Month + Day + WeekDay + Snow + Holiday", data_source))
+
+
+cnxn = pyodbc.connect(conn_str)
+cursor=cnxn.cursor()
+cursor.execute("INSERT INTO rental_models(model_name, lang, native_model) VALUES(?, ?, ?)", ("linear_model", "Python", trained_model))
+cnxn.commit()

ML Services/03.VS Code の利用/SimpleScoring.py

@@ -0,0 +1,42 @@
+from revoscalepy import rx_lin_mod, rx_dtree, rx_logit, rx_dforest
+from revoscalepy import rx_serialize_model, rx_summary
+from microsoftml import rx_fast_linear, rx_neural_network
+from microsoftml import rx_predict
+# from revoscalepy import rx_predict
+import pandas as pd
+import pyodbc
+import os
+import numpy as np
+
+model_file = "rx_fast_linear"
+
+conn_str = 'Driver=SQL Server;Server=<Server Name>;Database=MLDB;Uid=<User Name>;Pwd=<Password>;'
+cnxn = pyodbc.connect(conn_str)
+inputsql ='''
+SELECT C1, C2 FROM (
+VALUES
+('A',1),('B',2),('C',3),('D',4),('E',5),('F',6),('G',7),('H',8),('I',9),('J',10),
+('K',11),('L',12),('M',13),('N',14),('O',15),('P',16),('Q',17),('R',18),('S',19),('T',20),
+('U',21),('V',22),('W',23),('X',24),('Y',25),('Z',26)
+) AS T1(C1, C2)
+'''
+train_data = pd.read_sql(inputsql, cnxn)
+train_data["C1"] = train_data["C1"].astype("category")
+print(train_data["C1"] )
+
+# model = rx_lin_mod("C2 ~ C1", data = train_data)
+model = rx_fast_linear("C2 ~ C1", data = train_data, method = "regression")
+print(rx_summary("C2 ~ C1", train_data))
+
+trained_model = rx_serialize_model(model, realtime_scoring_only = True)
+# Dump learned model to file
+with open(r'c:\temp\trained_model_{0}.pickle'.format(model_file), mode='wb') as f:
+    f.write(trained_model)
+
+
+p_data = pd.DataFrame([0,5,10,99,4], columns=["C1"])
+p_data["C1"] = train_data["C1"].astype("category")
+print(p_data["C1"])
+
+pred = rx_predict(model, data = p_data)
+print(pred)

ML Services/04.Cognitive 連携/00.テストデータの追加.sql

@@ -0,0 +1,10 @@
+SET NOCOUNT ON
+DROP TABLE IF EXISTS BlobTable
+CREATE TABLE BlobTable (Filename nvarchar(255), Data varbinary(max))
+Truncate Table BlobTable
+GO
+
+DECLARE @blob varbinary(max)
+
+SELECT @blob = (SELECT * FROM OPENROWSET(BULK N'C:\Scripts\Target.png', SINGLE_BLOB) as tmp)
+INSERT INTO BlobTable VALUES('Target.png', @blob)

ML Services/04.Cognitive 連携/01.Face API による顔認識.sql

@@ -0,0 +1,66 @@
+USE [ML]
+GO
+
+DECLARE @filename nvarchar(255) = 'Target.png'
+
+DECLARE @image varbinary(max)
+DECLARE @face nvarchar(max)
+
+DECLARE @key varchar(32) = '<API Key>'
+DECLARE @baseurl varchar(max) = 'https://southeastasia.api.cognitive.microsoft.com/face/v1.0/'
+
+SELECT @image = Data FROM BlobTable WHERE Filename =@filename
+
+execute sp_execute_external_script 
+@language = N'Python',
+@script = N'
+import io
+import cognitive_face as CF
+import json
+from PIL import Image, ImageDraw
+
+CF.Key.set(key)
+CF.BaseUrl.set(baseurl)
+
+_f = io.BytesIO(image)
+
+_result = CF.face.detect(_f, attributes="age,smile,gender")
+
+face = str(_result)
+print(_result)
+
+if len(_result) != 0:
+    jsondata = json.loads(str(_result[0]).replace("''", "\""))
+
+    img = Image.open(_f)
+    draw = ImageDraw.Draw(img)
+    draw.rectangle((
+    (jsondata["faceRectangle"]["left"],
+    jsondata["faceRectangle"]["top"]),
+    (jsondata["faceRectangle"]["left"] + jsondata["faceRectangle"]["width"], 
+    jsondata["faceRectangle"]["top"] + jsondata["faceRectangle"]["height"])))
+    img.save(r"c:\scripts\face_detect.png")
+
+face = str(_result)
+', 
+@params = N'@key varchar(32), @baseurl varchar(max), @image varbinary(max), @face nvarchar(max) OUTPUT',
+@image = @image,
+@key = @key,
+@baseurl = @baseurl,
+@face = @face OUTPUT
+
+SELECT @filename AS filename, * FROM OPENJSON(replace(@face, '''', '"'))
+WITH(
+faceid uniqueidentifier '$.faceId',
+faceAttributes_gender varchar(20) '$.faceAttributes.gender',
+faceAttributes_smile float '$.faceAttributes.smile',
+faceAttributes_age float '$.faceAttributes.age',
+faceRectangle_Top int '$.faceRectangle.top',
+faceRectangle_left int '$.faceRectangle.left',
+faceRectangle_width int '$.faceRectangle.width',
+faceRectangle_height int '$.faceRectangle.height'
+)
+
+
+
+