from tracemalloc import start import elasticsearch from flask import Flask, request, make_response, jsonify from flask_httpauth import HTTPBasicAuth import logging import datetime from datetime import datetime, timedelta from dateutil import parser, tz import json from elasticsearch import Elasticsearch, helpers import re from configparser import ConfigParser import math import calendar logging.basicConfig(filename='/var/www/html/logs/deywuro_analytics.log', filemode='a', format='%(asctime)s %(name)s %(levelname)s %(message)s', level=logging.INFO) app = Flask(__name__) #Read config.ini file config_object = ConfigParser() config_object.read("config.ini") #Get info elasticinfo = config_object["elastic"] authinfo = config_object["auth"] auth = HTTPBasicAuth() USER_DATA = { str(authinfo["username"]): str(authinfo["password"]) } @auth.verify_password def verify(username, password): if not (username and password): return False return USER_DATA.get(username) == password # DAILY USER COUNTS @app.route('/user/counts/', methods=['POST', 'GET']) @auth.login_required def userCounts(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] num_cards = req['num_cards'] today = datetime.utcnow().date() if not num_cards.strip(): num_cards = 10 elif num_cards.lower().strip() == 'no limit': num_cards = 500 else: num_cards = int(num_cards) if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() json_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"2":{"terms":{"field":"username.keyword","size":"{}".format(num_cards),"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{}}}} try: log_res = es.search(index="deywuro-logs-*", body=json_payload) daily_counts = dict(by_key=log_res['aggregations']['2']['buckets']) return make_response(jsonify(code=200, daily_counts=daily_counts), 200) except Exception as e: logging.info(e) return make_response(jsonify(code=500, error='search api failed'), 200) # ACTIVE USERS @app.route('/active/users/', methods=['POST', 'GET']) @auth.login_required def activeUsers(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() request_start_date = req['start_date'] request_end_date = req['end_date'] today = datetime.utcnow().date() start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) end_date = start_date + timedelta(days=1, microseconds=-1) current_year = today.year current_month = today.month start_month = f"{current_year}-{'%02d'%current_month}-01T00:00:00" _end = calendar.monthrange(current_year, current_month)[1] end_month = f"{current_year}-{'%02d'%current_month}-{'%02d'%_end}T23:59:59" start_date = start_date.isoformat() end_date = end_date.isoformat() start_month = parser.parse(start_month).isoformat() end_month = parser.parse(end_month).isoformat() if not request_start_date.strip() or not request_end_date.strip(): request_start_date = datetime(today.year, today.month, today.day, tzinfo=tz.tzutc()) request_end_date = request_start_date + timedelta(days=1, microseconds=-1) interval = '1h' request_start_date = request_start_date.isoformat() request_end_date = request_end_date.isoformat() count_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs": {"type_count": {"cardinality": {"field": "username.keyword"}}} } deywuro_registered_payload = { "aggs": {"type_count": {"value_count": {"field": "id"}}} } new_daily_users_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(start_date),"lte":"{}".format(end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs": {"type_count": {"value_count": {"field": "id"}}} } new_users_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(start_month),"lte":"{}".format(end_month)}}}, {"query_string":{"query":"*"}}]}}, "aggs": {"type_count": {"value_count": {"field": "id"}}} } top_users_payload= { "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}} ]}}, "aggs":{"2":{"terms":{"field":"username.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"3" : {"sum":{"field" : "sms_count"}}}}} } api_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}], "must": [{"match": {"originated.keyword": "apiuser"}}] }}, "aggs":{"2":{"terms":{"field":"username.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"3" : {"sum":{"field" : "sms_count"}}}}} } bulk_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}], "must": [{"match": {"originated.keyword": "guiuser"}}] }}, "aggs":{"2":{"terms":{"field":"username.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"3" : {"sum":{"field" : "sms_count"}}}}} } smpp_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_time":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"2":{"terms":{"field":"esme_user.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"3" : {"sum":{"field" : "page_count"}}}}} } traffic_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_time":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"4":{"date_histogram":{"interval":"{}".format(interval),"field":"submit_time","min_doc_count":"0","extended_bounds":{"min":"{}".format(request_start_date),"max":"{}".format(request_end_date)}}, "aggs":{}}} } try: deywuro_active_users_res = es.search(index="deywuro-logs-*", body=count_payload) deywuro_active_users = deywuro_active_users_res["aggregations"]["type_count"]["value"] deywuro_registered_users_res = es.search(index="sms-users", body=deywuro_registered_payload) deywuro_registered_users = deywuro_registered_users_res["aggregations"]["type_count"]["value"] new_daily_users_res = es.search(index="sms-users", body=new_daily_users_payload) new_daily_users = new_daily_users_res["aggregations"]["type_count"]["value"] new_users_res = es.search(index="sms-users", body=new_users_payload) new_users = new_users_res["aggregations"]["type_count"]["value"] top_users_res = es.search(index="deywuro-logs-*", body=top_users_payload) top_users = dict(by_key=top_users_res['aggregations']['2']['buckets']) api_users_res = es.search(index="deywuro-logs-*", body=api_payload) api_users = dict(by_key=api_users_res['aggregations']['2']['buckets']) bulk_users_res = es.search(index="deywuro-logs-*", body=bulk_payload) bulk_users = dict(by_key=bulk_users_res['aggregations']['2']['buckets']) smpp_users_res = es.search(index="prod-smpp-sms-*", body=smpp_payload) smpp_users = dict(by_key=smpp_users_res['aggregations']['2']['buckets']) traffic_res = es.search(index="prod-kannel-access-*", body=traffic_payload) traffic = [dict(_time=parser.isoparse(traffic_res["aggregations"]["4"]["buckets"][i]["key_as_string"]).strftime('%H:%M'), count=traffic_res["aggregations"]["4"]["buckets"][i]["doc_count"]) for i in range(len(traffic_res["aggregations"]["4"]["buckets"]))] return make_response(jsonify(code=200, deywuro_active_users=deywuro_active_users, deywuro_registered_users=deywuro_registered_users, new_daily_users=new_daily_users, new_users = new_users, top_users=top_users, api_users=api_users, bulk_users=bulk_users, smpp_users=smpp_users, traffic=traffic), 200) except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search api failed'), 200) else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() # start_time = request_start_date.replace("T", " ") # end_time = request_end_date.replace("T", " ") # start_time = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') # end_time = datetime.strptime(end_time, '%Y-%m-%d %H:%M:%S') # time_range = abs(end_time - start_time).total_seconds() # max_data_points = 1006 # interval = math.floor(time_range/max_data_points) # interval = str(interval) + 's' interval = '1h' count_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs": {"type_count": {"cardinality": {"field": "username.keyword"}}} } deywuro_registered_payload = { "aggs": {"type_count": {"value_count": {"field": "id"}}} } new_daily_users_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(start_date),"lte":"{}".format(end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs": {"type_count": {"value_count": {"field": "id"}}} } new_users_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(start_month),"lte":"{}".format(end_month)}}}, {"query_string":{"query":"*"}}]}}, "aggs": {"type_count": {"value_count": {"field": "id"}}} } top_users_payload= { "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}} ]}}, "aggs":{"2":{"terms":{"field":"username.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"3" : {"sum":{"field" : "sms_count"}}}}} } api_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}], "must": [{"match": {"originated.keyword": "apiuser"}}] }}, "aggs":{"2":{"terms":{"field":"username.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"3" : {"sum":{"field" : "sms_count"}}}}} } bulk_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_date":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}], "must": [{"match": {"originated.keyword": "guiuser"}}] }}, "aggs":{"2":{"terms":{"field":"username.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"3" : {"sum":{"field" : "sms_count"}}}}} } smpp_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_time":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"2":{"terms":{"field":"esme_user.keyword","size":10,"order":{"_count":"desc"},"min_doc_count":0}, "aggs":{"3" : {"sum":{"field" : "page_count"}}}}} } traffic_payload = { "size":0, "query":{"bool":{"filter":[{"range":{"submit_time":{"gte":"{}".format(request_start_date),"lte":"{}".format(request_end_date)}}}, {"query_string":{"query":"*"}}]}}, "aggs":{"4":{"date_histogram":{"interval":"{}".format(interval),"field":"submit_time","min_doc_count":"0","extended_bounds":{"min":"{}".format(request_start_date),"max":"{}".format(request_end_date)}}, "aggs":{}}} } try: deywuro_active_users_res = es.search(index="deywuro-logs-*", body=count_payload) deywuro_active_users = deywuro_active_users_res["aggregations"]["type_count"]["value"] deywuro_registered_users_res = es.search(index="sms-users", body=deywuro_registered_payload) deywuro_registered_users = deywuro_registered_users_res["aggregations"]["type_count"]["value"] new_daily_users_res = es.search(index="sms-users", body=new_daily_users_payload) new_daily_users = new_daily_users_res["aggregations"]["type_count"]["value"] new_users_res = es.search(index="sms-users", body=new_users_payload) new_users = new_users_res["aggregations"]["type_count"]["value"] top_users_res = es.search(index="deywuro-logs-*", body=top_users_payload) top_users = dict(by_key=top_users_res['aggregations']['2']['buckets']) api_users_res = es.search(index="deywuro-logs-*", body=api_payload) api_users = dict(by_key=api_users_res['aggregations']['2']['buckets']) bulk_users_res = es.search(index="deywuro-logs-*", body=bulk_payload) bulk_users = dict(by_key=bulk_users_res['aggregations']['2']['buckets']) smpp_users_res = es.search(index="prod-smpp-sms-*", body=smpp_payload) smpp_users = dict(by_key=smpp_users_res['aggregations']['2']['buckets']) traffic_res = es.search(index="prod-kannel-access-*", body=traffic_payload) traffic = [dict(_time=parser.isoparse(traffic_res["aggregations"]["4"]["buckets"][i]["key_as_string"]).strftime('%m/%d %H:%M'), count=traffic_res["aggregations"]["4"]["buckets"][i]["doc_count"]) for i in range(len(traffic_res["aggregations"]["4"]["buckets"]))] return make_response(jsonify(code=200, deywuro_active_users=deywuro_active_users, deywuro_registered_users=deywuro_registered_users, new_daily_users=new_daily_users, new_users = new_users, top_users=top_users, api_users=api_users, bulk_users=bulk_users, smpp_users=smpp_users, traffic=traffic), 200) except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='search api failed'), 200) if __name__ == '__main__': app.run(host='0.0.0.0')