import elasticsearch from flask import Flask, request, make_response, jsonify from flask.json import JSONDecoder from flask_httpauth import HTTPBasicAuth import logging import datetime from datetime import date, datetime, timedelta import json from elasticsearch import Elasticsearch, helpers import re from configparser import ConfigParser from dateutil import parser, tz import numpy as np logging.basicConfig(filename='/var/www/html/logs/general_kedebah_api.log', filemode='a', format='%(asctime)s %(name)s %(levelname)s %(message)s', level=logging.INFO) app = Flask(__name__) #Read config.ini file config_object = ConfigParser() config_object.read("config.ini") #Get info elasticinfo = config_object["elastic"] authinfo = config_object["auth"] auth = HTTPBasicAuth() USER_DATA = { str(authinfo["username"]): str(authinfo["password"]) } @auth.verify_password def verify(username, password): if not (username and password): return False return USER_DATA.get(username) == password @app.route('/registered/companies/', methods=['POST', 'GET']) @auth.login_required def registered_companies(): es = Elasticsearch(str(elasticinfo["local"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() try: # registered companies json_payload = { "size" : 0, "aggs" : {"id_count" : { "value_count" : { "field" : "id.keyword" } } }} reg_comp_res = es.search(index="kedebah-tenants-*", body=json_payload) registered_companies = reg_comp_res['aggregations']['id_count']['value'] except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search failed'), 200) return make_response(jsonify(code=200, registered_companies=registered_companies), 200) @app.route('/active/companies/', methods=['POST', 'GET']) @auth.login_required def active_companies(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() try: # active companies json_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_datetime":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"2":{"date_histogram":{"interval":"365d","field":"submit_datetime","min_doc_count":0,"extended_bounds":{"min":"{}".format(request_start_date),"max":"{}".format(request_end_date)}}, "aggs":{"1":{"cardinality":{"field":"company_name.keyword"}}}}}} ac_res = es.search(index="kedebah-usage-monitoring-data-*", body=json_payload) ac_count = ac_res['aggregations']['2']['buckets'][0]['1']['value'] except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search failed'), 200) return make_response(jsonify(code=200, active_companies=ac_count), 200) @app.route('/kedebah/company-hits/', methods=['POST', 'GET']) @auth.login_required def kedebah_usage(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() try: # kedebah monitoring usage json_payload = { "size":0,"query":{"bool":{"filter":[{"range":{"submit_datetime":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"3":{"terms":{"field":"company_name.keyword","size":500,"order":{"_count":"desc"},"min_doc_count":1}, "aggs":{"2":{"date_histogram":{"interval":"365d","field":"submit_datetime","min_doc_count":"1","extended_bounds":{"min":"{}".format(request_start_date),"max":"{}".format(request_end_date)}}, "aggs":{}}}}}} monitor_res = es.search(index="kedebah-usage-monitoring-data-*", body=json_payload) kedebah_usage = dict(by_key=monitor_res['aggregations']['3']['buckets']) except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search failed'), 200) return make_response(jsonify(code=200, kedebah_company_hits=kedebah_usage), 200) @app.route('/unique/comp-count/', methods=['POST', 'GET']) @auth.login_required def uniqueCompanyCount(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() try: # active kedebah modules by unique company count json_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_datetime":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"3":{"terms":{"field":"module.keyword","size":10,"order":{"_key":"desc"},"min_doc_count":1}, "aggs":{"2":{"date_histogram":{"interval":"365d","field":"submit_datetime","min_doc_count":0,"extended_bounds":{"min":"{}".format(request_start_date),"max":"{}".format(request_end_date)}}, "aggs":{"1":{"cardinality":{"field":"company_name.keyword"}}}}}}}} unique_company_res = es.search(index="kedebah-usage-monitoring-data-*", body=json_payload) unique_company_count = dict(by_key=unique_company_res['aggregations']['3']['buckets']) except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search failed'), 200) return make_response(jsonify(code=200, unique_company_count=unique_company_count), 200) @app.route('/user/engagement/', methods=['POST', 'GET']) @auth.login_required def user_engagement(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() try: # user_engagement json_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_datetime":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"2":{"date_histogram":{"interval":"30s","field":"submit_datetime","min_doc_count":0,"extended_bounds":{"min":"{}".format(request_start_date),"max":"{}".format(request_end_date)}}, "aggs":{}}}} user_eng_res = es.search(index="kedebah-usage-monitoring-data-*", body=json_payload) user_eng = dict(by_key=user_eng_res['aggregations']['2']['buckets']) except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search failed'), 200) return make_response(jsonify(code=200, user_engagement=user_eng), 200) @app.route('/unique/users-count/', methods=['POST', 'GET']) @auth.login_required def uniqueUsersCount(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() try: # active kedebah modules by unique users count json_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_datetime":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"3":{"terms":{"field":"module.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"2":{"date_histogram":{"interval":"365d","field":"submit_datetime","min_doc_count":"0","extended_bounds":{"min":"{}".format(request_start_date),"max":"{}".format(request_end_date)}}, "aggs":{"1":{"cardinality":{"field":"company_user_id.keyword"}}}}}}}} users_unique_res = es.search(index="kedebah-usage-monitoring-data-*", body=json_payload) users_unique_count = dict(by_key=users_unique_res['aggregations']['3']['buckets']) except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search failed'), 200) return make_response(jsonify(code=200, unique_users_count=users_unique_count), 200) @app.route('/kedebah/errors/', methods=['POST', 'GET']) @auth.login_required def kedebah_errors(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() try: # kedebah errors json_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_time":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"3":{"terms":{"field":"error_name.keyword","size":500,"order":{"_count":"desc"},"min_doc_count":1}, "aggs":{"2":{"date_histogram":{"interval":"1m","field":"submit_time","min_doc_count":"1","extended_bounds":{"min":"{}".format(request_start_date),"max":"{}".format(request_end_date)}}, "aggs":{}}}}}} kedebah_errors_res = es.search(index="kedebah-lareval-error-report-*", body=json_payload) error_counts = {} error_values = {} error_keys = [kedebah_errors_res['aggregations']['3']['buckets'][i]['key'] for i in range(len(kedebah_errors_res['aggregations']['3']['buckets']))] i = 0 for error_key in error_keys: for j in range(len(kedebah_errors_res['aggregations']['3']['buckets'][i]['2']['buckets'])): if not error_key in error_counts.keys(): error_counts[error_key] = [(kedebah_errors_res['aggregations']['3']['buckets'][i]['2']['buckets'][j]['doc_count'], kedebah_errors_res['aggregations']['3']['buckets'][i]['2']['buckets'][j]['key_as_string'])] continue error_counts[error_key].append((kedebah_errors_res['aggregations']['3']['buckets'][i]['2']['buckets'][j]['doc_count'], kedebah_errors_res['aggregations']['3']['buckets'][i]['2']['buckets'][j]['key_as_string'])) counts = len(error_counts[error_key]) error_values[error_key] = dict(counts=counts, error_counts_values=error_counts[error_key]) i += 1 t0 = [kedebah_errors_res['aggregations']['3']['buckets'][i]['2']['buckets'][j]['key_as_string'] for i in range(len(kedebah_errors_res['aggregations']['3']['buckets'])) for j in range(len(kedebah_errors_res['aggregations']['3']['buckets'][i]['2']['buckets']))] period = np.unique(np.array(t0), axis=0).tolist() except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search failed'), 200) return make_response(jsonify(code=200, error_keys=error_keys, error_values=error_values, period=period), 200) @app.route('/kedebah/worldmap/', methods=['POST', 'GET']) @auth.login_required def kedebah_worldmap(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() try: # world map json_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_datetime":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}]}}, "aggs":{"3":{"terms":{"field":"location.lat"}, "aggs":{"4":{"terms":{"field":"location.lon"}}}}}} world_map_res = es.search(index="kedebah-usage-monitoring-data-*", body=json_payload) world_map_count = [dict(counts=world_map_res['aggregations']['3']['buckets'][i]['doc_count'], lat=world_map_res['aggregations']['3']['buckets'][i]['key'], long=world_map_res['aggregations']['3']['buckets'][i]['4']['buckets'][j]['key']) for i in range(len(world_map_res['aggregations']['3']['buckets'])) for j in range(len(world_map_res['aggregations']['3']['buckets'][i]['4']['buckets']))] except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search failed'), 200) return make_response(jsonify(code=200, world_map_count=world_map_count), 200) if __name__ == '__main__': app.run(host='0.0.0.0')