import glob
import argparse
import re
import locale
import subprocess
from subprocess import PIPE
import sys
import os
import time
import json
import sqlite3
from googletrans import Translator 
import urllib2
from mtranslate.core import translate
from bs4 import BeautifulSoup
from selenium import webdriver
import shutil
from shutil import copyfile
sys.path.append("../../apicog_py")

from datainitializer.ScreenApiFlow import ScreenApiFlow
from datainitializer.ImageString import ImageString
from config.constant import Values

def get_full_name_of_aty(atyname):
	pkgname, atyname = atyname.split('/')
	if atyname.startswith('.'):
		atyname = pkgname+atyname
	return pkgname,atyname

def get_jsonstr_from_jsonfile(image_json,json_path):
	resouces_ids = set()
	view_texts= set()
	translator = Translator()
	cls_names= set()

	for view in image_json['views']:
		if 'resource_id' in view and view['resource_id']:
			text = view['resource_id'].split('/')[-1]
			try:
				resource_en = translate(text,"en","auto")
			except urllib2.HTTPError as e:
				if 'Request Entity Too Large' in str(e):
					resource_en = ''
				else:
					os.system("echo "+json_path+" >> ./HTTPError.txt")
			resouces_ids.add(resource_en)

		if 'text' in view and view['text']:
			print view['text']
			# text_en = translator.translate(view['text']).text
			text_en = ''
			try:
				text_en = translate(view['text'],"en","auto")
			except Exception as e:
				if 'Request Entity Too Large' in str(e):
					text_en = ''
				else:
					os.system("echo "+json_path+" >> ./HTTPError.txt")
			view_texts.add(text_en)

		if 'class' in view and view['class']:
			if view['class'].startswith('android.view') or view['class'].startswith('android.widget'):
				continue
			print view['class']
			# text_en = translator.translate(view['text']).text
			text_en = ''
			try:
				text_en = translate(view['class'],"en","auto")
			except Exception as e:
				if 'Request Entity Too Large' in str(e):
					text_en = ''
				else:
					os.system("echo "+json_path+" >> ./HTTPError.txt")
			cls_names.add(text_en)

	jsonstr = '+*-+*-'.join(resouces_ids)+ '+*-+*-' +'+*-+*-'.join(view_texts) + '+*-+*-' +'+*-+*-'.join(cls_names)
	return jsonstr

def create_image_string():
	saf_db = ScreenApiFlow(Values.screen_api_flow_db)
	saf_db.initialize()
	appresultpaths = saf_db.get_appresultpaths()
	saf_db.finalize()

	is_db = ImageString(Values.image_to_string_db)
	is_db.initialize()
	


	index = 0
	appresultpaths = set(appresultpaths)
	app_droidbot_paths = [os.path.join(path,'droidbot/states') for path in appresultpaths]
	

	
	for droidbot_state in app_droidbot_paths:

		screenimagepaths = glob.glob(os.path.join(droidbot_state, '*.jpg'))
		
		for imagepath in screenimagepaths:

			print index, '--',imagepath
			json_name = imagepath.split('/')[-1].replace('screen','state')
			json_name = json_name[:-3]+'json'
			json_path = os.path.join('/'.join(imagepath.split('/')[:-1]), json_name)
			index+=1

			if is_db.get_json_string(imagepath):
				continue

			# if not 'a49a8e28ddb48297bcfe0393d40d5e7b0cdb836a6d7b3a8adc6d95a3c7a4991a' in json_path:
			# 	continue
			with open(json_path, 'r') as input_file:
				image_json = json.load(input_file)
			image_json = byteify(image_json)

			imageaty = image_json['foreground_activity']
			pkgname,full_name_aty = get_full_name_of_aty(imageaty)
			jsonstr = get_jsonstr_from_jsonfile(image_json,json_path)
			item = {'imagepath':imagepath, 'imagepkg':pkgname, 'imageaty':full_name_aty, 'jsonstr':jsonstr, 'screenshotstr':None, 'googleimagestr':None, 'msazurestr':None, 'tineyestr':None, 'sametoid':None}
			if pkgname in Values.syspkgs:
				item['sametoid'] = -1
			is_db.process_item(item)

	is_db.finalize()


def state_json_string_extract():

	is_db = ImageString(Values.image_to_string_db)
	is_db.initialize()
	screenimagepaths = is_db.get_image_path()

	index = 0
	screenimagepaths = set(screenimagepaths)
	print len(screenimagepaths)
	for imagepath in screenimagepaths:

		print index, '--',imagepath
		json_name = imagepath.split('/')[-1].replace('screen','state')
		json_name = json_name[:-3]+'json'
		json_path = os.path.join('/'.join(imagepath.split('/')[:-1]), json_name)
		index+=1

		# if is_db.get_json_string(imagepath):
		# 	continue

		# if not 'a49a8e28ddb48297bcfe0393d40d5e7b0cdb836a6d7b3a8adc6d95a3c7a4991a' in json_path:
		# 	continue
		with open(json_path, 'r') as input_file:
			image_json = json.load(input_file)
		image_json = byteify(image_json)
		jsonstr = get_jsonstr_from_jsonfile(image_json,json_path)
		# print jsonstr
		is_db.update_json_str(imagepath,jsonstr)

	is_db.finalize()

def byteify(input, encoding='utf-8'):
    if isinstance(input, dict):
        return {byteify(key): byteify(value) for key, value in input.iteritems()}
    elif isinstance(input, list):
        return [byteify(element) for element in input]
    elif isinstance(input, unicode):
        return input.encode(encoding)
    else:
        return input


def tineye_image_topic():
	from strfilters.language_filter import LanguageFilter
	tineye_url = 'https://tineye.com/search/?url=http://www.cse.unsw.edu.au/~jieliu/image_api/'

	is_db = ImageString(Values.image_to_string_db)
	is_db.initialize()
	screenimagepaths = is_db.get_image_path()
	index = 0
	print len(screenimagepaths)

	for imagepath in screenimagepaths:
		image_name = imagepath.split('/')[-1]
		complete_url = tineye_url + image_name
		index+=1
		print index, '--',imagepath

		if is_db.get_tineye_str(imagepath):
			continue

		tineye_str = ''
		try:
			# query the website and return the html to the variable 'page'
			page = urllib2.urlopen(complete_url)

			# parse the html using beautiful soup and store in variable 'soup'
			soup = BeautifulSoup(page, "lxml")
			for tag in soup.find_all('div', attrs={'class':'match'}):
				if tag.get_text():
					tineye_str += tag.get_text().encode('utf-8').strip()+' '
		except urllib2.HTTPError, e:
			print(str(e.code))

		# strs = [s.strip() for s in strs.encode('utf-8').split('\n')]
		# strs = ' '.join(filter(None, strs))
		# print strs
		# screenshot_str = LanguageFilter().filter(strs)
		# if screenshot_str:
		# 	screenshot_str = ' '.join(screenshot_str).encode('utf-8')
		# 	print 'LF--------------',screenshot_str
		# 	rule = re.compile(ur"[^a-zA-Z.,?!\'\"]")
		# 	line = rule.sub(' ',screenshot_str)
		# 	strr = list()
		# 	for s in filter(None, line.split(' ')):
		# 		rule = re.compile(ur"[^a-zA-Z]")
		# 		eng = rule.sub('',s)
		# 		if len(eng)>1 and eng != 'AM' and eng !='PM':
		# 			strr.append(s)
		# 	screenshot_str = ' '.join(strr)
		
		# 	if screenshot_str:
		# 		print 're--------------',screenshot_str
		# 		is_db.update_tineye_str(imagepath,screenshot_str)
		if tineye_str:
			print tineye_str.split('\n\n')
			break
	is_db.finalize()

def google_image_topic():
	page_url = 'https://www.google.com/searchbyimage?site=search&sa=X&image_url=http://www.cse.unsw.edu.au/~jieliu/image_api/'


	is_db = ImageString(Values.image_to_string_db)
	is_db.initialize()
	screenimagepaths = is_db.get_image_path()
	index = 0
	print len(screenimagepaths)
	browser = webdriver.Firefox()

	for imagepath in screenimagepaths:
		image_name = imagepath.split('/')[-1]
		complete_url = page_url + image_name

		print index, '--',complete_url
		index+=1

		if is_db.get_googleimagestr(imagepath):
			continue


		
		browser.get(complete_url)
		a=[]
		a = browser.find_elements_by_class_name("fKDtNb");
		texts= set()
		for te in a:
			try:
				print te.text.encode('utf-8')
				text_en = translate(te.text.encode('utf-8'),"en","auto")
			except urllib2.HTTPError as e:
				if 'Request Entity Too Large' in str(e):
					text_en = ''
				else:
					os.system("echo "+json_path+" >> ./HTTPError.txt")
			texts.add(text_en)

		if texts:
			print '--------------',', '.join(texts)
			image_topic_str = ', '.join(texts)
			is_db.update_googleimagestr(imagepath,image_topic_str)


	is_db.finalize()

	browser.close()
	
def screenshot_strs():
	try:
	    import Image
	except ImportError:
	    from PIL import Image
	import pytesseract
	from strfilters.language_filter import LanguageFilter

	is_db = ImageString(Values.image_to_string_db)
	is_db.initialize()
	screenimagepaths = is_db.get_image_path()
	index = 0
	print len(screenimagepaths)

	for imagepath in screenimagepaths:
		# if '/fdroidnew/es.esy.CosyDVR/droidbot/states/screen_2018-12-06_092829.jpg' not in imagepath:
		# 	continue
		index+=1
		old_imagepath = imagepath
		print index, '--',imagepath
		# json_string = is_db.get_json_string(imagepath)
		if 'fdroidnew' in imagepath or 'malwarenew' in imagepath:
			imagepath = imagepath.replace('.jpg','.png')
			if not os.path.isfile(imagepath):
				imagepath = imagepath.replace('.png','.jpg')
		# elif is_db.get_screenshot_str(imagepath):
		# 	continue

		try:
			strs = pytesseract.image_to_string(Image.open(imagepath))
		except Exception:
			strs=''
		screen_str = list()
		strs = [s.strip() for s in strs.encode('utf-8').split('\n')]
		for fstrr in filter(None, strs):
			screenshot_str = LanguageFilter().filter(fstrr)
			if screenshot_str:
				screenshot_str = ' '.join(screenshot_str).encode('utf-8')
				# print 'LF--------------',screenshot_str
				rule = re.compile(ur"[^a-zA-Z.,?!\'\"]")
				line = rule.sub(' ',screenshot_str)
				strr = list()
				for s in filter(None, line.split(' ')):
					rule = re.compile(ur"[^a-zA-Z]")
					eng = rule.sub('',s)
					if len(eng)>1 and eng != 'AM' and eng !='PM':
						strr.append(s)
				screenshot_str = ' '.join(strr)
		
			if screenshot_str:
				if screenshot_str.lower() != 'all':
					screen_str.append(screenshot_str)
		print 're--------------',', '.join(screen_str)
		if screen_str:
			is_db.update_screenshot_str(old_imagepath,', '.join(screen_str))
		else:
			is_db.update_screenshot_str(old_imagepath,'+*-+*-')
	is_db.finalize()



if __name__ == "__main__":

	create_image_string()
	# state_json_string_extract()
	# tineye_image_topic()
	# google_image_topic()
	screenshot_strs()
