From ad880cb4c4ea0653151d5ddfd81ca7ee302eb788 Mon Sep 17 00:00:00 2001 From: codingcrush Date: Wed, 24 Oct 2018 11:31:09 +0800 Subject: [PATCH 1/2] fix: syntax optimization --- app/controller/api/views.py | 3 +- app/controller/common/render.py | 5 +- app/controller/manage.py | 1 + app/dao/time_series_detector/anomaly_op.py | 2 +- app/dao/time_series_detector/sample_op.py | 2 +- .../time_series_detector/algorithm/gbdt.py | 2 +- .../algorithm/xgboosting.py | 8 +-- .../feature/classification_features.py | 7 ++- .../feature/statistical_features.py | 63 ++++++++++--------- .../time_series_detector/sample_service.py | 6 +- app/utils/utils.py | 34 +++++----- tests/fixtures.py | 1 + tests/test_common.py | 1 + tests/test_feature.py | 2 + 14 files changed, 72 insertions(+), 65 deletions(-) diff --git a/app/controller/api/views.py b/app/controller/api/views.py index dc8c61c..d27c87f 100644 --- a/app/controller/api/views.py +++ b/app/controller/api/views.py @@ -12,6 +12,7 @@ from app.service.time_series_detector.detect_service import * from app.config.errorcode import * from app.utils.utils import * + def check_post(func): @wraps(func) def f(request): @@ -69,7 +70,7 @@ def download_sample(request): response['Content-Type'] = 'application/octet-stream' response['Content-Disposition'] = 'attachment;filename = "SampleExport.csv"' return response - except Exception, ex: + except Exception as ex: return_dict = build_ret_data(THROW_EXP, str(ex)) return render_json(return_dict) else: diff --git a/app/controller/common/render.py b/app/controller/common/render.py index d388a66..c03e057 100644 --- a/app/controller/common/render.py +++ b/app/controller/common/render.py @@ -2,9 +2,10 @@ import json from django.http import HttpResponse -def render_json(dictionary={}): +def render_json(dictionary=None): + dictionary = {} if dictionary is None else dictionary response = HttpResponse(json.dumps(dictionary), content_type="application/json") response['Access-Control-Allow-Origin'] = '*' response["Access-Control-Allow-Headers"] = "Origin, X-Requested-With, Content-Type" response["Access-Control-Allow-Methods"] = "GET, POST, PUT, OPTIONS" - return response \ No newline at end of file + return response diff --git a/app/controller/manage.py b/app/controller/manage.py index 3f98e69..14c5d38 100644 --- a/app/controller/manage.py +++ b/app/controller/manage.py @@ -22,6 +22,7 @@ def wait_child(signum, frame): raise print('handle SIGCHLD end') + if __name__ == "__main__": signal.signal(signal.SIGCHLD, wait_child) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings.settings") diff --git a/app/dao/time_series_detector/anomaly_op.py b/app/dao/time_series_detector/anomaly_op.py index 8cc658d..e702442 100644 --- a/app/dao/time_series_detector/anomaly_op.py +++ b/app/dao/time_series_detector/anomaly_op.py @@ -81,7 +81,7 @@ class AbnormalOperation(object): def update_anomaly(self, data): update_str = "UPDATE anomaly set mark_flag = %s where id = %s" params = [data['markFlag'], data['id']] - record_num = self.__cur.execute(update_str, params) + self.__cur.execute(update_str, params) self.__conn.commit() if MARK_NEGATIVE == data['markFlag'] or MARK_POSITIVE == data['markFlag']: diff --git a/app/dao/time_series_detector/sample_op.py b/app/dao/time_series_detector/sample_op.py index fcddeb7..37ae8f2 100644 --- a/app/dao/time_series_detector/sample_op.py +++ b/app/dao/time_series_detector/sample_op.py @@ -191,7 +191,7 @@ class SampleOperation(object): item_per_page = data['itemPerPage'] request_page = data['requestPage'] beg_limit = (item_per_page * (request_page - 1)) - limit = (item_per_page) + limit = item_per_page params = [] query_str = "" diff --git a/app/service/time_series_detector/algorithm/gbdt.py b/app/service/time_series_detector/algorithm/gbdt.py index 2482ed7..7a20365 100644 --- a/app/service/time_series_detector/algorithm/gbdt.py +++ b/app/service/time_series_detector/algorithm/gbdt.py @@ -82,7 +82,7 @@ class Gbdt(object): grd.fit(X_train, y_train) model_name = MODEL_PATH + task_id + "_model" joblib.dump(grd, model_name) - except Exception, ex: + except Exception as ex: return TRAIN_ERR, str(ex) return OP_SUCCESS, "" diff --git a/app/service/time_series_detector/algorithm/xgboosting.py b/app/service/time_series_detector/algorithm/xgboosting.py index 31cb402..3b956c9 100644 --- a/app/service/time_series_detector/algorithm/xgboosting.py +++ b/app/service/time_series_detector/algorithm/xgboosting.py @@ -74,7 +74,7 @@ class XGBoosting(object): """ try: f = open(feature_file_name, "w") - except Exception, ex: + except Exception as ex: return CAL_FEATURE_ERR, str(ex) times = 0 for temp in data: @@ -104,7 +104,7 @@ class XGBoosting(object): features.append(temp) try: ret_code, ret_data = self.__save_libsvm_format(features, feature_file_name) - except Exception, ex: + except Exception as ex: ret_code = CAL_FEATURE_ERR ret_data = str(ex) return ret_code, ret_data @@ -124,7 +124,7 @@ class XGBoosting(object): return ret_code, ret_data try: dtrain = xgb.DMatrix(feature_file_name) - except Exception, ex: + except Exception as ex: return READ_FEATURE_FAILED, str(ex) params = { 'max_depth': self.max_depth, @@ -141,7 +141,7 @@ class XGBoosting(object): try: bst = xgb.train(params, dtrain, num_round) bst.save_model(model_name) - except Exception, ex: + except Exception as ex: return TRAIN_ERR, str(ex) return OP_SUCCESS, "" diff --git a/app/service/time_series_detector/feature/classification_features.py b/app/service/time_series_detector/feature/classification_features.py index fee4fda..e7ddf5b 100644 --- a/app/service/time_series_detector/feature/classification_features.py +++ b/app/service/time_series_detector/feature/classification_features.py @@ -78,9 +78,10 @@ def time_series_binned_entropy(x): def get_classification_features(x): - classification_features = [] - classification_features.append(time_series_autocorrelation(x)) - classification_features.append(time_series_coefficient_of_variation(x)) + classification_features = [ + time_series_autocorrelation(x), + time_series_coefficient_of_variation(x) + ] classification_features.extend(time_series_binned_entropy(x)) # append yourself classification features here... diff --git a/app/service/time_series_detector/feature/statistical_features.py b/app/service/time_series_detector/feature/statistical_features.py index f800b32..9ec4d3c 100644 --- a/app/service/time_series_detector/feature/statistical_features.py +++ b/app/service/time_series_detector/feature/statistical_features.py @@ -410,37 +410,38 @@ def time_series_range(x): def get_statistical_features(x): - statistical_features = [] - statistical_features.append(time_series_maximum(x)) - statistical_features.append(time_series_minimum(x)) - statistical_features.append(time_series_mean(x)) - statistical_features.append(time_series_variance(x)) - statistical_features.append(time_series_standard_deviation(x)) - statistical_features.append(time_series_skewness(x)) - statistical_features.append(time_series_kurtosis(x)) - statistical_features.append(time_series_median(x)) - statistical_features.append(time_series_abs_energy(x)) - statistical_features.append(time_series_absolute_sum_of_changes(x)) - statistical_features.append(time_series_variance_larger_than_std(x)) - statistical_features.append(time_series_count_above_mean(x)) - statistical_features.append(time_series_count_below_mean(x)) - statistical_features.append(time_series_first_location_of_maximum(x)) - statistical_features.append(time_series_first_location_of_minimum(x)) - statistical_features.append(time_series_last_location_of_maximum(x)) - statistical_features.append(time_series_last_location_of_minimum(x)) - statistical_features.append(int(time_series_has_duplicate(x))) - statistical_features.append(int(time_series_has_duplicate_max(x))) - statistical_features.append(int(time_series_has_duplicate_min(x))) - statistical_features.append(time_series_longest_strike_above_mean(x)) - statistical_features.append(time_series_longest_strike_below_mean(x)) - statistical_features.append(time_series_mean_abs_change(x)) - statistical_features.append(time_series_mean_change(x)) - statistical_features.append(time_series_percentage_of_reoccurring_datapoints_to_all_datapoints(x)) - statistical_features.append(time_series_ratio_value_number_to_time_series_length(x)) - statistical_features.append(time_series_sum_of_reoccurring_data_points(x)) - statistical_features.append(time_series_sum_of_reoccurring_values(x)) - statistical_features.append(time_series_sum_values(x)) - statistical_features.append(time_series_range(x)) + statistical_features = [ + time_series_maximum(x), + time_series_minimum(x), + time_series_mean(x), + time_series_variance(x), + time_series_standard_deviation(x), + time_series_skewness(x), + time_series_kurtosis(x), + time_series_median(x), + time_series_abs_energy(x), + time_series_absolute_sum_of_changes(x), + time_series_variance_larger_than_std(x), + time_series_count_above_mean(x), + time_series_count_below_mean(x), + time_series_first_location_of_maximum(x), + time_series_first_location_of_minimum(x), + time_series_last_location_of_maximum(x), + time_series_last_location_of_minimum(x), + int(time_series_has_duplicate(x)), + int(time_series_has_duplicate_max(x)), + int(time_series_has_duplicate_min(x)), + time_series_longest_strike_above_mean(x), + time_series_longest_strike_below_mean(x), + time_series_mean_abs_change(x), + time_series_mean_change(x), + time_series_percentage_of_reoccurring_datapoints_to_all_datapoints(x), + time_series_ratio_value_number_to_time_series_length(x), + time_series_sum_of_reoccurring_data_points(x), + time_series_sum_of_reoccurring_values(x), + time_series_sum_values(x), + time_series_range(x) + ] # append yourself statistical features here... return statistical_features diff --git a/app/service/time_series_detector/sample_service.py b/app/service/time_series_detector/sample_service.py index 737cc3d..5668ff4 100644 --- a/app/service/time_series_detector/sample_service.py +++ b/app/service/time_series_detector/sample_service.py @@ -62,7 +62,7 @@ class SampleService(object): elif row[6] == "negative": negative_count = negative_count + 1 count = count + 1 - except Exception, ex: + except Exception as ex: traceback.print_exc() return_dict = build_ret_data(FILE_FORMAT_ERR, str(ex)) return return_dict @@ -82,12 +82,12 @@ class SampleService(object): return self.__sample.query_sample(json.loads(body)) def sample_download(self, body): - ret_data = "" + ret_code = 1000 try: if len(body) > VALUE_LEN_MAX: return "" ret_code, ret_data = self.__sample.download_sample(body) - except Exception, ex: + except Exception as ex: traceback.print_exc() ret_data = build_ret_data(THROW_EXP, str(ex)) return ret_code, ret_data diff --git a/app/utils/utils.py b/app/utils/utils.py index ede81c9..c95771d 100644 --- a/app/utils/utils.py +++ b/app/utils/utils.py @@ -27,10 +27,7 @@ def is_standard_time_series(time_series, window=180): :return: True or False :return type: boolean """ - if len(time_series) == 5 * window + 3 and np.mean(time_series[(4 * window + 2):]) > 0: - return True - else: - return False + return bool(len(time_series) == 5 * window + 3 and np.mean(time_series[(4 * window + 2):]) > 0) def split_time_series(time_series, window=180): @@ -46,12 +43,13 @@ def split_time_series(time_series, window=180): data_b_left = time_series[(2 * window + 1):(3 * window + 2)] data_b_right = time_series[(3 * window + 1):(4 * window + 2)] data_a = time_series[(4 * window + 2):] - split_time_series = [] - split_time_series.append(data_c_left) - split_time_series.append(data_c_right) - split_time_series.append(data_b_left) - split_time_series.append(data_b_right) - split_time_series.append(data_a) + split_time_series = [ + data_c_left, + data_c_right, + data_b_left, + data_b_right, + data_a + ] return split_time_series @@ -75,12 +73,13 @@ def normalize_time_series(split_time_series): normalized_data_b_left = split_time_series[2] normalized_data_b_right = split_time_series[3] normalized_data_a = split_time_series[4] - normalized_split_time_series = [] - normalized_split_time_series.append(normalized_data_c_left) - normalized_split_time_series.append(normalized_data_c_right) - normalized_split_time_series.append(normalized_data_b_left) - normalized_split_time_series.append(normalized_data_b_right) - normalized_split_time_series.append(normalized_data_a) + normalized_split_time_series = [ + normalized_data_c_left, + normalized_data_c_right, + normalized_data_b_left, + normalized_data_b_right, + normalized_data_a + ] return normalized_split_time_series @@ -94,9 +93,8 @@ def exce_service(func): try: ret_code, ret_data = func(*args, **kwargs) return_dict = build_ret_data(ret_code, ret_data) - except Exception, ex: + except Exception as ex: traceback.print_exc() return_dict = build_ret_data(THROW_EXP, str(ex)) return return_dict return wrapper - diff --git a/tests/fixtures.py b/tests/fixtures.py index 0ecb2b5..19035f7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -11,6 +11,7 @@ Unless required by applicable law or agreed to in writing, software distributed from unittest import TestCase import random + class DataTestCase(TestCase): def create_test_data_a(self): return [850600,889768,883237,896313,870407,868385,865300,889802,894983,836835,937571,904475,892846,878769,886624,892638,894804,889133,908860, diff --git a/tests/test_common.py b/tests/test_common.py index dfd9a3f..b38827f 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -12,6 +12,7 @@ from tests.fixtures import DataTestCase from app.config.common import * from app.utils.utils import * + class CommonTestCase(DataTestCase): def test_check_value(self): diff --git a/tests/test_feature.py b/tests/test_feature.py index c3388e1..c3fd583 100644 --- a/tests/test_feature.py +++ b/tests/test_feature.py @@ -19,10 +19,12 @@ class FeatureTestCase(DataTestCase): self.assertTrue(time_series_maximum(testdata_a) == 1020900) self.assertTrue(time_series_minimum(testdata_a) == 824757) self.assertTrue((time_series_mean(testdata_a) - 919324.34) < 1e-2) + def test_two(self): x = "hello" assert 'hello' in x + if __name__ == '__main__': a = FeatureTestCase() a.test_features() From b85efc8b61d7e1329f354f0751ed09b492d6b16f Mon Sep 17 00:00:00 2001 From: lxd1190 <34830669+lxd1190@users.noreply.github.com> Date: Wed, 24 Oct 2018 15:39:46 +0800 Subject: [PATCH 2/2] Delete test_common.py do not need this file anymore --- tests/test_common.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 tests/test_common.py diff --git a/tests/test_common.py b/tests/test_common.py deleted file mode 100644 index b38827f..0000000 --- a/tests/test_common.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Tencent is pleased to support the open source community by making Metis available. -Copyright (C) 2018 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the BSD 3-Clause License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at -https://opensource.org/licenses/BSD-3-Clause -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" - -from tests.fixtures import DataTestCase -from app.config.common import * -from app.utils.utils import * - - -class CommonTestCase(DataTestCase): - - def test_check_value(self): - str_param = self.generate_random_str(INPUT_LEN_ENG_MAX+1) - self.assertTrue(validate_value(str_param) != 0 ) - list_param = [] - for i in range(0,INPUT_LIST_LEN_MAX): - list_param.append(self.generate_random_str(INPUT_LEN_ENG_MAX)) - self.assertTrue(validate_value(list_param) == 0 ) - list_param.append(self.generate_random_str(INPUT_LEN_ENG_MAX)) - self.assertFalse(validate_value(list_param) == 0 )