From e53c9cdfd26e33c7191c0ee7cf43baacf224aebc Mon Sep 17 00:00:00 2001 From: Komei Shimamura Date: Sun, 27 Sep 2015 23:34:06 +0900 Subject: [PATCH 1/2] Add type and params fields to ComponentSerializer Effecting APIs: - /api/v1/oprations/:ComponentName/:id - /api/v1/oprations/:id Related-Issue: #34 --- cognitive/app/models.py | 2 ++ cognitive/app/serializers.py | 23 ++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cognitive/app/models.py b/cognitive/app/models.py index b285222..0768ba9 100644 --- a/cognitive/app/models.py +++ b/cognitive/app/models.py @@ -54,6 +54,8 @@ class Data_operation_type(models.Model): function_arg = models.CharField(max_length=50, choices=FUNCTION_ARG) function_arg_id = models.CharField(max_length=50, blank=True, null=True) function_subtype = models.CharField(max_length=50, choices=FUNCTION_SUBTYPE) + + # This should be JSON format function_subtype_arg = models.CharField(max_length=50, blank=True, null=True) def __str__(self): diff --git a/cognitive/app/serializers.py b/cognitive/app/serializers.py index 13bde0e..742bbfb 100644 --- a/cognitive/app/serializers.py +++ b/cognitive/app/serializers.py @@ -12,8 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +import json + from rest_framework import serializers -from .models import User, Experiment, Component, Workflow +from .models import User, Experiment, Component, Workflow, Data_operation_type class UserSerializer(serializers.ModelSerializer): @@ -35,11 +37,30 @@ class Meta: # 'execution_start_time', 'execution_end_time', 'component_start_id') +class ComponentOutputValue(serializers.ModelSerializer): + class Meta: + fields = ('function_subtype', 'function_subtype_arg') + model = Data_operation_type + class ComponentSerializer(serializers.ModelSerializer): + type = serializers.SerializerMethodField('component_type') + params = serializers.SerializerMethodField('component_params') class Meta: model = Component + def component_type(self, obj): + return obj.operation_type.function_subtype + + def component_params(self, obj): + data = obj.operation_type.function_subtype_arg + if data is None: + return '' + elif data.startswith(('[', '{')): + return json.loads(data) + else: + # TODO(|Less Priority| All Data_operation_type.function_subtype_arg values should be JSON) + return data class WorkflowSerializer(serializers.ModelSerializer): From 072ec66f2c7f6dce9a19140e5f182a4db912a18b Mon Sep 17 00:00:00 2001 From: Komei Shimamura Date: Mon, 28 Sep 2015 00:26:39 +0900 Subject: [PATCH 2/2] Add outputs field to Component model Related-Issue: #34 --- cognitive/app/models.py | 2 + cognitive/app/serializers.py | 2 + .../experiment/directive/node.directive.js | 4 +- cognitive/app/view/view_operation.py | 17 +++++- cognitive/app/view/view_result_local.py | 54 +++++++++---------- requirements.txt | 7 +-- 6 files changed, 52 insertions(+), 34 deletions(-) diff --git a/cognitive/app/models.py b/cognitive/app/models.py index 0768ba9..66ac89b 100644 --- a/cognitive/app/models.py +++ b/cognitive/app/models.py @@ -15,6 +15,7 @@ from django.db import models from django.core.validators import validate_email from django.core.exceptions import ValidationError +from jsonfield import JSONField import string import random @@ -169,6 +170,7 @@ class Component(models.Model): execution_end_time = models.DateField(blank=True, null=True) data_location = models.CharField(max_length=50, blank=True, null=True) preferred_data_location = models.CharField(max_length=50, blank=True, null=True) + outputs = JSONField() # component_id = models.IntegerField(blank=True, null=True) diff --git a/cognitive/app/serializers.py b/cognitive/app/serializers.py index 742bbfb..8232b70 100644 --- a/cognitive/app/serializers.py +++ b/cognitive/app/serializers.py @@ -42,6 +42,7 @@ class Meta: fields = ('function_subtype', 'function_subtype_arg') model = Data_operation_type + class ComponentSerializer(serializers.ModelSerializer): type = serializers.SerializerMethodField('component_type') params = serializers.SerializerMethodField('component_params') @@ -62,6 +63,7 @@ def component_params(self, obj): # TODO(|Less Priority| All Data_operation_type.function_subtype_arg values should be JSON) return data + class WorkflowSerializer(serializers.ModelSerializer): class Meta: diff --git a/cognitive/app/static/app/whiteboard/experiment/directive/node.directive.js b/cognitive/app/static/app/whiteboard/experiment/directive/node.directive.js index 0c4c716..43a3743 100644 --- a/cognitive/app/static/app/whiteboard/experiment/directive/node.directive.js +++ b/cognitive/app/static/app/whiteboard/experiment/directive/node.directive.js @@ -63,8 +63,8 @@ .on("dragend", finishDrawingConnection)); g.append('text') - .attr('x', '{{node.x - 15}}') - .attr('y', '{{node.y - 5}}') + .attr('ng-attr-x', '{{node.x - 15}}') + .attr('ng-attr-y', '{{node.y - 5}}') .attr('font-family', 'FontAwesome') .attr('class', 'node close-icon') .attr('node', '{{node.id}}') diff --git a/cognitive/app/view/view_operation.py b/cognitive/app/view/view_operation.py index 8f093ab..e42570d 100644 --- a/cognitive/app/view/view_operation.py +++ b/cognitive/app/view/view_operation.py @@ -19,10 +19,19 @@ from rest_framework.renderers import JSONRenderer from pandas import read_csv, datetime import json +import re import urllib2 MAX_COMPONENTS_PER_EXP = 100 +def get_csv_fields(csv_data): + for line in re.split('\n|\r', csv_data): + if line == '': + continue + parsed_line = line.split(',') + return [{'id': i,'name': parsed_line[i]} for i in range(len(parsed_line))] + return [] + ##### # Operation function_type function_arg function_subtype function_arg_id function_subtype_arg # mathformula Update {comp_type} {op_type} {comp_id} {op_constant} @@ -35,7 +44,6 @@ # input Create Table Input - {filename} # machine learning Create Model {model_type} {Train-test} {ML arguments} - class OperationViewSet(viewsets.ViewSet): def set_operation(self, operation, data): @@ -178,9 +186,14 @@ def create(self, request, operation): print "Experiment ", exp_id, " Operation ", operation op = self.set_operation(operation, data) + outputs = [] + if operation == 'input' and data["input_file_type"] == "csv": + outputs = get_csv_fields(data["data_values"]) + component = Component( experiment=exp, created_time=datetime.now(), - modified_time=datetime.now(), operation_type=op) + modified_time=datetime.now(), operation_type=op, outputs=outputs) + component.save() serializer = ComponentSerializer(component) return send_response("GET", serializer) diff --git a/cognitive/app/view/view_result_local.py b/cognitive/app/view/view_result_local.py index dbda4f2..7030733 100644 --- a/cognitive/app/view/view_result_local.py +++ b/cognitive/app/view/view_result_local.py @@ -112,10 +112,10 @@ def run(self): input_data, model_type, train_data_percentage, input_features, target_feature) output_data = classifier.learn() - except ValueError as e: + except ValueError: status = "failure" err_msg = " Invalid input for the model training" - except KeyError as e: + except KeyError: status = "failure" err_msg = target_feature + " column is not available for Model Training" @@ -132,15 +132,15 @@ def run(self): column_id = float(op.function_arg_id) column_name = feature_names[column_id] if column_name not in input_data: - #print "Column name ", column_name, " not present. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " not present. Skipping" + # continue # throw error in module status status = "failure" err_msg = column_name + " column is not available for current operation" elif input_data[column_name].dtype == 'object': - #print "Column name ", column_name, " is not integer/float. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " is not integer/float. Skipping" + # continue # throw error in module status status = "failure" - err_msg = " Invalid input in column "+ column_name+ " for the current operation" + err_msg = " Invalid input in column " + column_name + " for the current operation" else: input_data[column_name] += constant_value if op.function_subtype == 'Sub': @@ -148,15 +148,15 @@ def run(self): column_id = float(op.function_arg_id) column_name = feature_names[column_id] if column_name not in input_data: - #print "Column name ", column_name, " not present. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " not present. Skipping" + # continue # throw error in module status status = "failure" err_msg = column_name + " column is not available for current operation" elif input_data[column_name].dtype == 'object': - #print "Column name ", column_name, " is not integer/float. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " is not integer/float. Skipping" + # continue # throw error in module status status = "failure" - err_msg = " Invalid input in column "+ column_name+ " for the current operation" + err_msg = " Invalid input in column " + column_name + " for the current operation" else: input_data[column_name] -= constant_value if op.function_subtype == 'Mult': @@ -164,15 +164,15 @@ def run(self): column_id = float(op.function_arg_id) column_name = feature_names[column_id] if column_name not in input_data: - #print "Column name ", column_name, " not present. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " not present. Skipping" + # continue # throw error in module status status = "failure" err_msg = column_name + " column is not available for current operation" elif input_data[column_name].dtype == 'object': - #print "Column name ", column_name, " is not integer/float. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " is not integer/float. Skipping" + # continue # throw error in module status status = "failure" - err_msg = " Invalid input in column "+ column_name+ " for the current operation" + err_msg = " Invalid input in column " + column_name + " for the current operation" else: input_data[column_name] *= constant_value if op.function_subtype == 'Div': @@ -180,15 +180,15 @@ def run(self): column_id = float(op.function_arg_id) column_name = feature_names[column_id] if column_name not in input_data: - #print "Column name ", column_name, " not present. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " not present. Skipping" + # continue # throw error in module status status = "failure" err_msg = column_name + " column is not available for current operation" elif input_data[column_name].dtype == 'object': - #print "Column name ", column_name, " is not integer/float. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " is not integer/float. Skipping" + # continue # throw error in module status status = "failure" - err_msg = " Invalid input in column "+ column_name+ " for the current operation" + err_msg = " Invalid input in column " + column_name + " for the current operation" else: input_data[column_name] /= constant_value if op.function_subtype == 'Normalize': @@ -196,8 +196,8 @@ def run(self): column_name = feature_names[column_id] sum_array = input_data.sum(axis=0) if column_name not in sum_array: - #print "Column name ", column_name, " not present. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " not present. Skipping" + # continue # throw error in module status status = "failure" err_msg = column_name + " column is not available for current operation" else: @@ -222,8 +222,8 @@ def run(self): for elem in column_id_list: column_name = feature_names[elem] if column_name not in input_data: - #print "Column name ", column_name, " not present. Skipping" - #continue # throw error in module status + # print "Column name ", column_name, " not present. Skipping" + # continue # throw error in module status status = "failure" err_msg = column_name + " column is not available for current operation" else: @@ -330,7 +330,7 @@ def run(self): if self.cache_results is True: CACHE[self.experiment] = input_data - #print self.result + # print self.result print self.result["status"] print self.result["message"] break diff --git a/requirements.txt b/requirements.txt index e22e3ce..993de73 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,15 @@ -Django==1.7.1 +Django>=1.7.1 django-rest-swagger>=0.2.9 Markdown==2.5.1 Pygments==2.0.1 -django-bower==5.0.1 +django-bower>=5.0.1 django-mptt==0.6.1 django-treebeard==2.0 djangorestframework==2.4.4 -hamlpy==0.82.2 +hamlpy>=0.82.2 jasmine==2.3.0 jasmine-core==2.3.4 +jsonfield>=1.0.3 numpy==1.9.1 pandas==0.15.1 python-dateutil==2.2