from flask import Flask, request, make_response, jsonify from flask_httpauth import HTTPBasicAuth import logging import datetime from datetime import datetime, timedelta from dateutil import parser, tz from elasticsearch import Elasticsearch from configparser import ConfigParser from pandasticsearch import Select import pandas as pd logging.basicConfig(filename='/var/www/html/logs/credits_histories.log', filemode='a', format='%(asctime)s %(name)s %(levelname)s %(message)s', level=logging.INFO) app = Flask(__name__) #Read config.ini file config_object = ConfigParser() config_object.read("config.ini") #Get info elasticinfo = config_object["elastic"] authinfo = config_object["auth"] savepathinfo = config_object["savepath"] auth = HTTPBasicAuth() USER_DATA = { str(authinfo["username"]): str(authinfo["password"]) } @auth.verify_password def verify(username, password): if not (username and password): return False return USER_DATA.get(username) == password # paginate credit_histories @app.route('/credit_histories/', methods=['POST', 'GET']) @auth.login_required def creditHistories(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() username = str(req['username']) request_start_date = req['start_date'] request_end_date = req['end_date'] if not request_start_date.strip() or not request_end_date.strip(): if not req['cursor'].strip(): json_payload = { "size": 10, "query": { "match": {"username.keyword":username} }, "sort": [ { "created_at" : "desc"} ] } else: cursor = int(req['cursor']) json_payload = { "size": 10, "query": { "match": {"username.keyword":username} }, "search_after" : [cursor], "sort": [ { "created_at" : "desc"} ] } count_payload = { "query": { "match": {"username.keyword":username} } } else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() if not req['cursor'].strip(): json_payload = { "size": 10, "query": { "bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(request_start_date), "lt":"{}".format(request_end_date)}}}, {"match": {"username.keyword":username}}]} }, "sort": [ { "created_at" : "desc"} ] } else: cursor = int(req['cursor']) json_payload = { "size": 10, "query": { "bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(request_start_date), "lt":"{}".format(request_end_date)}}}, {"match": {"username.keyword":username}}]} }, "search_after" : [cursor], "sort": [ { "created_at" : "desc"} ] } count_payload = { "query": { "bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(request_start_date), "lt":"{}".format(request_end_date)}}}, {"match": {"username.keyword":username}}]} }, } results = [] try: count = es.count(index="hellio-credit-histories", body=count_payload)['count'] log_res = es.search(index="hellio-credit-histories", body=json_payload) for doc in log_res['hits']['hits']: results.append(dict(id=doc['_source']['id'], user_id=doc['_source']['user_id'], username=doc['_source']['username'], price=doc['_source']['price'], credit_alloted=doc['_source']['credit_alloted'], old_bal=doc['_source']['old_bal'], new_bal=doc['_source']['new_bal'], credit_used=doc['_source']['credit_used'], credited=doc['_source']['credited'], currency=doc['_source']['currency'], credited_by=doc['_source']['credited_by'], comments=doc['_source']['comments'], status=doc['_source']['status'], date_credited=doc['_source']['date_credited'], created_at=doc['_source']['created_at'], updated_at=doc['_source']['updated_at'], deleted_at=doc['_source']['deleted_at'] )) try: cursor = log_res['hits']['hits'][-1]['sort'][0] logging.info(f"sort value:{cursor}") except: cursor = 0 logging.info(f"sort value:{cursor}") if not len(results): return make_response(jsonify(code=404, error='No search results'), 200) else: logging.info('Results found') logging.info(results[0]) return make_response(jsonify(code=200, cursor=cursor, doc_count=count, results=results), 200) except Exception as e: logging.info(f'search failed with error {e}') return make_response(jsonify(code=500, error='search api failed'), 200) # download credit_histories @app.route('/download/credit_histories/', methods=['POST', 'GET']) @auth.login_required def downloadCreditHistories(): es = Elasticsearch(str(elasticinfo["live"])) if request.method == 'POST': req = request.get_json() username = req['username'] request_start_date = req['start_date'] request_end_date = req['end_date'] if not request_start_date.strip() or not request_end_date.strip(): json_payload = { "size": 10000, "query": { "match": {"username.keyword":username} }, "sort": [ { "created_at" : "desc"} ] } count_payload = { "query": { "match": {"username.keyword":username} }, } else: request_start_date = parser.parse(req['start_date']).isoformat() request_end_date = parser.parse(req['end_date']).isoformat() json_payload = { "size": 10000, "query": { "bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(request_start_date), "lt":"{}".format(request_end_date)}}}, {"match": {"username.keyword":username}}]} }, "sort": [ { "created_at" : "desc"} ] } count_payload = { "query": { "bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(request_start_date), "lte":"{}".format(request_end_date)}}}, {"match": {"username.keyword":username}}]} }, } try: count = es.count(index="hellio-credit-histories", body=count_payload)['count'] res = es.search(index="hellio-credit-histories", body=json_payload) df_1 = Select.from_dict(res).to_pandas() logging.info(f'count:{count}') if not len(res['hits']['hits']): return make_response(jsonify(code=404, error='No search results'), 200) if count <= 10000: req_time = datetime.utcnow().strftime('%Y%m%d%H%M') save_path = "{}{}-credits-{}.xlsx".format(savepathinfo['savepath'], username, req_time) logging.info(f'Length of dataframe: {len(df_1)}') df_1.to_excel(save_path, index=True) return make_response(jsonify(code=200, file_path=save_path), 200) else: while len(res['hits']['hits']): try: cursor = res['hits']['hits'][-1]['sort'][0] json_payload = { "size": 10000, "query": { "bool":{"filter":[{"range":{"created_at":{"gte":"{}".format(request_start_date), "lt":"{}".format(request_end_date)}}}, {"match": {"username.keyword":username}}]} }, "search_after" : [cursor], "sort": [ { "created_at" : "desc"} ] } res = es.search(index="hellio-credit-histories", body=json_payload) df_2 = Select.from_dict(res).to_pandas() df_1 = pd.concat([df_1, df_2], ignore_index=True) except: logging.info(f'Length of dataframe: {len(df_1)}') req_time = datetime.utcnow().strftime('%Y%m%d%H%M') save_path = "{}{}-credits-{}.xlsx".format(savepathinfo['savepath'], username, req_time) df_1.to_excel(save_path, index=True) break return make_response(jsonify(code=200, file_path=save_path), 200) except Exception as e: logging.info(f'failed with error {e}') return make_response(jsonify(code=500, error='failed to download file'), 200) if __name__ == '__main__': app.run(host='0.0.0.0')